1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe Endpoint controller driver 4 * 5 * Copyright (C) 2017 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 #include <linux/align.h> 10 #include <linux/bitfield.h> 11 #include <linux/of.h> 12 #include <linux/overflow.h> 13 #include <linux/platform_device.h> 14 15 #include "pcie-designware.h" 16 #include <linux/pci-epc.h> 17 #include <linux/pci-epf.h> 18 19 /** 20 * dw_pcie_ep_get_func_from_ep - Get the struct dw_pcie_ep_func corresponding to 21 * the endpoint function 22 * @ep: DWC EP device 23 * @func_no: Function number of the endpoint device 24 * 25 * Return: struct dw_pcie_ep_func if success, NULL otherwise. 26 */ 27 struct dw_pcie_ep_func * 28 dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) 29 { 30 struct dw_pcie_ep_func *ep_func; 31 32 list_for_each_entry(ep_func, &ep->func_list, list) { 33 if (ep_func->func_no == func_no) 34 return ep_func; 35 } 36 37 return NULL; 38 } 39 40 static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, 41 enum pci_barno bar, int flags) 42 { 43 struct dw_pcie_ep *ep = &pci->ep; 44 u32 reg; 45 46 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 47 dw_pcie_dbi_ro_wr_en(pci); 48 dw_pcie_ep_writel_dbi2(ep, func_no, reg, 0x0); 49 dw_pcie_ep_writel_dbi(ep, func_no, reg, 0x0); 50 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 51 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0x0); 52 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0x0); 53 } 54 dw_pcie_dbi_ro_wr_dis(pci); 55 } 56 57 /** 58 * dw_pcie_ep_reset_bar - Reset endpoint BAR 59 * @pci: DWC PCI device 60 * @bar: BAR number of the endpoint 61 */ 62 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) 63 { 64 u8 func_no, funcs; 65 66 funcs = pci->ep.epc->max_functions; 67 68 for (func_no = 0; func_no < funcs; func_no++) 69 __dw_pcie_ep_reset_bar(pci, func_no, bar, 0); 70 } 71 EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); 72 73 static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) 74 { 75 return PCI_FIND_NEXT_CAP(dw_pcie_ep_read_cfg, PCI_CAPABILITY_LIST, 76 cap, NULL, ep, func_no); 77 } 78 79 static u16 dw_pcie_ep_find_ext_capability(struct dw_pcie_ep *ep, 80 u8 func_no, u8 cap) 81 { 82 return PCI_FIND_NEXT_EXT_CAP(dw_pcie_ep_read_cfg, 0, 83 cap, NULL, ep, func_no); 84 } 85 86 static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 87 struct pci_epf_header *hdr) 88 { 89 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 90 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 91 92 dw_pcie_dbi_ro_wr_en(pci); 93 dw_pcie_ep_writew_dbi(ep, func_no, PCI_VENDOR_ID, hdr->vendorid); 94 dw_pcie_ep_writew_dbi(ep, func_no, PCI_DEVICE_ID, hdr->deviceid); 95 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_REVISION_ID, hdr->revid); 96 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CLASS_PROG, hdr->progif_code); 97 dw_pcie_ep_writew_dbi(ep, func_no, PCI_CLASS_DEVICE, 98 hdr->subclass_code | hdr->baseclass_code << 8); 99 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CACHE_LINE_SIZE, 100 hdr->cache_line_size); 101 dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_VENDOR_ID, 102 hdr->subsys_vendor_id); 103 dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_ID, hdr->subsys_id); 104 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_INTERRUPT_PIN, 105 hdr->interrupt_pin); 106 dw_pcie_dbi_ro_wr_dis(pci); 107 108 return 0; 109 } 110 111 /* BAR Match Mode inbound iATU mapping */ 112 static int dw_pcie_ep_ib_atu_bar(struct dw_pcie_ep *ep, u8 func_no, int type, 113 dma_addr_t parent_bus_addr, enum pci_barno bar, 114 size_t size) 115 { 116 int ret; 117 u32 free_win; 118 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 119 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 120 121 if (!ep_func) 122 return -EINVAL; 123 124 if (!ep_func->bar_to_atu[bar]) 125 free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); 126 else 127 free_win = ep_func->bar_to_atu[bar] - 1; 128 129 if (free_win >= pci->num_ib_windows) { 130 dev_err(pci->dev, "No free inbound window\n"); 131 return -EINVAL; 132 } 133 134 ret = dw_pcie_prog_ep_inbound_atu(pci, func_no, free_win, type, 135 parent_bus_addr, bar, size); 136 if (ret < 0) { 137 dev_err(pci->dev, "Failed to program IB window\n"); 138 return ret; 139 } 140 141 /* 142 * Always increment free_win before assignment, since value 0 is used to identify 143 * unallocated mapping. 144 */ 145 ep_func->bar_to_atu[bar] = free_win + 1; 146 set_bit(free_win, ep->ib_window_map); 147 148 return 0; 149 } 150 151 static void dw_pcie_ep_clear_ib_maps(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) 152 { 153 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 154 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 155 struct device *dev = pci->dev; 156 unsigned int i, num; 157 u32 atu_index; 158 u32 *indexes; 159 160 if (!ep_func) 161 return; 162 163 /* Tear down the BAR Match Mode mapping, if any. */ 164 if (ep_func->bar_to_atu[bar]) { 165 atu_index = ep_func->bar_to_atu[bar] - 1; 166 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); 167 clear_bit(atu_index, ep->ib_window_map); 168 ep_func->bar_to_atu[bar] = 0; 169 return; 170 } 171 172 /* Tear down all Address Match Mode mappings, if any. */ 173 indexes = ep_func->ib_atu_indexes[bar]; 174 num = ep_func->num_ib_atu_indexes[bar]; 175 ep_func->ib_atu_indexes[bar] = NULL; 176 ep_func->num_ib_atu_indexes[bar] = 0; 177 if (!indexes) 178 return; 179 for (i = 0; i < num; i++) { 180 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, indexes[i]); 181 clear_bit(indexes[i], ep->ib_window_map); 182 } 183 devm_kfree(dev, indexes); 184 } 185 186 static u64 dw_pcie_ep_read_bar_assigned(struct dw_pcie_ep *ep, u8 func_no, 187 enum pci_barno bar, int flags) 188 { 189 u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 190 u32 lo, hi; 191 u64 addr; 192 193 lo = dw_pcie_ep_readl_dbi(ep, func_no, reg); 194 195 if (flags & PCI_BASE_ADDRESS_SPACE) 196 return lo & PCI_BASE_ADDRESS_IO_MASK; 197 198 addr = lo & PCI_BASE_ADDRESS_MEM_MASK; 199 if (!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) 200 return addr; 201 202 hi = dw_pcie_ep_readl_dbi(ep, func_no, reg + 4); 203 return addr | ((u64)hi << 32); 204 } 205 206 static int dw_pcie_ep_validate_submap(struct dw_pcie_ep *ep, 207 const struct pci_epf_bar_submap *submap, 208 unsigned int num_submap, size_t bar_size) 209 { 210 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 211 u32 align = pci->region_align; 212 size_t off = 0; 213 unsigned int i; 214 size_t size; 215 216 if (!align || !IS_ALIGNED(bar_size, align)) 217 return -EINVAL; 218 219 /* 220 * The submap array order defines the BAR layout (submap[0] starts 221 * at offset 0 and each entry immediately follows the previous 222 * one). Here, validate that it forms a strict, gapless 223 * decomposition of the BAR: 224 * - each entry has a non-zero size 225 * - sizes, implicit offsets and phys_addr are aligned to 226 * pci->region_align 227 * - each entry lies within the BAR range 228 * - the entries exactly cover the whole BAR 229 * 230 * Note: dw_pcie_prog_inbound_atu() also checks alignment for the 231 * PCI address and the target phys_addr, but validating up-front 232 * avoids partially programming iATU windows in vain. 233 */ 234 for (i = 0; i < num_submap; i++) { 235 size = submap[i].size; 236 237 if (!size) 238 return -EINVAL; 239 240 if (!IS_ALIGNED(size, align) || !IS_ALIGNED(off, align)) 241 return -EINVAL; 242 243 if (!IS_ALIGNED(submap[i].phys_addr, align)) 244 return -EINVAL; 245 246 if (off > bar_size || size > bar_size - off) 247 return -EINVAL; 248 249 off += size; 250 } 251 if (off != bar_size) 252 return -EINVAL; 253 254 return 0; 255 } 256 257 /* Address Match Mode inbound iATU mapping */ 258 static int dw_pcie_ep_ib_atu_addr(struct dw_pcie_ep *ep, u8 func_no, int type, 259 const struct pci_epf_bar *epf_bar) 260 { 261 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 262 const struct pci_epf_bar_submap *submap = epf_bar->submap; 263 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 264 enum pci_barno bar = epf_bar->barno; 265 struct device *dev = pci->dev; 266 u64 pci_addr, parent_bus_addr; 267 u64 size, base, off = 0; 268 int free_win, ret; 269 unsigned int i; 270 u32 *indexes; 271 272 if (!ep_func || !epf_bar->num_submap || !submap || !epf_bar->size) 273 return -EINVAL; 274 275 ret = dw_pcie_ep_validate_submap(ep, submap, epf_bar->num_submap, 276 epf_bar->size); 277 if (ret) 278 return ret; 279 280 base = dw_pcie_ep_read_bar_assigned(ep, func_no, bar, epf_bar->flags); 281 if (!base) { 282 dev_err(dev, 283 "BAR%u not assigned, cannot set up sub-range mappings\n", 284 bar); 285 return -EINVAL; 286 } 287 288 indexes = devm_kcalloc(dev, epf_bar->num_submap, sizeof(*indexes), 289 GFP_KERNEL); 290 if (!indexes) 291 return -ENOMEM; 292 293 ep_func->ib_atu_indexes[bar] = indexes; 294 ep_func->num_ib_atu_indexes[bar] = 0; 295 296 for (i = 0; i < epf_bar->num_submap; i++) { 297 size = submap[i].size; 298 parent_bus_addr = submap[i].phys_addr; 299 300 if (off > (~0ULL) - base) { 301 ret = -EINVAL; 302 goto err; 303 } 304 305 pci_addr = base + off; 306 off += size; 307 308 free_win = find_first_zero_bit(ep->ib_window_map, 309 pci->num_ib_windows); 310 if (free_win >= pci->num_ib_windows) { 311 ret = -ENOSPC; 312 goto err; 313 } 314 315 ret = dw_pcie_prog_inbound_atu(pci, free_win, type, 316 parent_bus_addr, pci_addr, size); 317 if (ret) 318 goto err; 319 320 set_bit(free_win, ep->ib_window_map); 321 indexes[i] = free_win; 322 ep_func->num_ib_atu_indexes[bar] = i + 1; 323 } 324 return 0; 325 err: 326 dw_pcie_ep_clear_ib_maps(ep, func_no, bar); 327 return ret; 328 } 329 330 static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, 331 struct dw_pcie_ob_atu_cfg *atu) 332 { 333 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 334 u32 free_win; 335 int ret; 336 337 free_win = find_first_zero_bit(ep->ob_window_map, pci->num_ob_windows); 338 if (free_win >= pci->num_ob_windows) { 339 dev_err(pci->dev, "No free outbound window\n"); 340 return -EINVAL; 341 } 342 343 atu->index = free_win; 344 ret = dw_pcie_prog_outbound_atu(pci, atu); 345 if (ret) 346 return ret; 347 348 set_bit(free_win, ep->ob_window_map); 349 ep->outbound_addr[free_win] = atu->parent_bus_addr; 350 351 return 0; 352 } 353 354 static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 355 struct pci_epf_bar *epf_bar) 356 { 357 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 358 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 359 enum pci_barno bar = epf_bar->barno; 360 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 361 362 if (!ep_func || !ep_func->epf_bar[bar]) 363 return; 364 365 __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); 366 367 dw_pcie_ep_clear_ib_maps(ep, func_no, bar); 368 369 ep_func->epf_bar[bar] = NULL; 370 } 371 372 static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie_ep *ep, u8 func_no, 373 enum pci_barno bar) 374 { 375 u32 reg, bar_index; 376 unsigned int offset, nbars; 377 int i; 378 379 offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); 380 if (!offset) 381 return offset; 382 383 reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 384 nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); 385 386 for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { 387 reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 388 bar_index = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, reg); 389 if (bar_index == bar) 390 return offset; 391 } 392 393 return 0; 394 } 395 396 static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, 397 struct pci_epf_bar *epf_bar) 398 { 399 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 400 enum pci_barno bar = epf_bar->barno; 401 size_t size = epf_bar->size; 402 int flags = epf_bar->flags; 403 u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 404 unsigned int rebar_offset; 405 u32 rebar_cap, rebar_ctrl; 406 int ret; 407 408 rebar_offset = dw_pcie_ep_get_rebar_offset(ep, func_no, bar); 409 if (!rebar_offset) 410 return -EINVAL; 411 412 ret = pci_epc_bar_size_to_rebar_cap(size, &rebar_cap); 413 if (ret) 414 return ret; 415 416 dw_pcie_dbi_ro_wr_en(pci); 417 418 /* 419 * A BAR mask should not be written for a resizable BAR. The BAR mask 420 * is automatically derived by the controller every time the "selected 421 * size" bits are updated, see "Figure 3-26 Resizable BAR Example for 422 * 32-bit Memory BAR0" in DWC EP databook 5.96a. We simply need to write 423 * BIT(0) to set the BAR enable bit. 424 */ 425 dw_pcie_ep_writel_dbi2(ep, func_no, reg, BIT(0)); 426 dw_pcie_ep_writel_dbi(ep, func_no, reg, flags); 427 428 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 429 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0); 430 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0); 431 } 432 433 /* 434 * Bits 31:0 in PCI_REBAR_CAP define "supported sizes" bits for sizes 435 * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes" 436 * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB. 437 */ 438 rebar_ctrl = dw_pcie_ep_readl_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL); 439 rebar_ctrl &= ~GENMASK(31, 16); 440 dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); 441 442 /* 443 * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically 444 * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR 445 * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a. 446 */ 447 dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CAP, rebar_cap); 448 449 dw_pcie_dbi_ro_wr_dis(pci); 450 451 return 0; 452 } 453 454 static int dw_pcie_ep_set_bar_programmable(struct dw_pcie_ep *ep, u8 func_no, 455 struct pci_epf_bar *epf_bar) 456 { 457 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 458 enum pci_barno bar = epf_bar->barno; 459 size_t size = epf_bar->size; 460 int flags = epf_bar->flags; 461 u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 462 463 dw_pcie_dbi_ro_wr_en(pci); 464 465 dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1)); 466 dw_pcie_ep_writel_dbi(ep, func_no, reg, flags); 467 468 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 469 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1)); 470 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0); 471 } 472 473 dw_pcie_dbi_ro_wr_dis(pci); 474 475 return 0; 476 } 477 478 static enum pci_epc_bar_type dw_pcie_ep_get_bar_type(struct dw_pcie_ep *ep, 479 enum pci_barno bar) 480 { 481 const struct pci_epc_features *epc_features; 482 483 if (!ep->ops->get_features) 484 return BAR_PROGRAMMABLE; 485 486 epc_features = ep->ops->get_features(ep); 487 488 return epc_features->bar[bar].type; 489 } 490 491 static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 492 struct pci_epf_bar *epf_bar) 493 { 494 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 495 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 496 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 497 enum pci_barno bar = epf_bar->barno; 498 size_t size = epf_bar->size; 499 enum pci_epc_bar_type bar_type; 500 int flags = epf_bar->flags; 501 int ret, type; 502 503 if (!ep_func) 504 return -EINVAL; 505 506 /* 507 * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs 508 * 1 and 2 to form a 64-bit BAR. 509 */ 510 if ((flags & PCI_BASE_ADDRESS_MEM_TYPE_64) && (bar & 1)) 511 return -EINVAL; 512 513 /* 514 * Certain EPF drivers dynamically change the physical address of a BAR 515 * (i.e. they call set_bar() twice, without ever calling clear_bar(), as 516 * calling clear_bar() would clear the BAR's PCI address assigned by the 517 * host). 518 */ 519 if (ep_func->epf_bar[bar]) { 520 /* 521 * We can only dynamically change a BAR if the new BAR size and 522 * BAR flags do not differ from the existing configuration. 523 * 524 * Note: this safety check only works when the caller uses 525 * a new struct pci_epf_bar in the second set_bar() call. 526 * If the same instance is updated in place and passed in, 527 * we cannot reliably detect invalid barno/size/flags 528 * changes here. 529 */ 530 if (ep_func->epf_bar[bar]->barno != bar || 531 ep_func->epf_bar[bar]->size != size || 532 ep_func->epf_bar[bar]->flags != flags) 533 return -EINVAL; 534 535 /* 536 * When dynamically changing a BAR, tear down any existing 537 * mappings before re-programming. This is redundant when 538 * both the old and new mappings are BAR Match Mode, but 539 * required to handle in-place updates and match-mode 540 * changes reliably. 541 */ 542 dw_pcie_ep_clear_ib_maps(ep, func_no, bar); 543 544 /* 545 * When dynamically changing a BAR, skip writing the BAR reg, as 546 * that would clear the BAR's PCI address assigned by the host. 547 */ 548 goto config_atu; 549 } else { 550 /* 551 * Subrange mapping is an update-only operation. The BAR 552 * must have been configured once without submaps so that 553 * subsequent set_bar() calls can update inbound mappings 554 * without touching the BAR register (and clobbering the 555 * host-assigned address). 556 */ 557 if (epf_bar->num_submap) 558 return -EINVAL; 559 } 560 561 bar_type = dw_pcie_ep_get_bar_type(ep, bar); 562 switch (bar_type) { 563 case BAR_FIXED: 564 /* 565 * There is no need to write a BAR mask for a fixed BAR (except 566 * to write 1 to the LSB of the BAR mask register, to enable the 567 * BAR). Write the BAR mask regardless. (The fixed bits in the 568 * BAR mask register will be read-only anyway.) 569 */ 570 fallthrough; 571 case BAR_PROGRAMMABLE: 572 ret = dw_pcie_ep_set_bar_programmable(ep, func_no, epf_bar); 573 break; 574 case BAR_RESIZABLE: 575 ret = dw_pcie_ep_set_bar_resizable(ep, func_no, epf_bar); 576 break; 577 default: 578 ret = -EINVAL; 579 dev_err(pci->dev, "Invalid BAR type\n"); 580 break; 581 } 582 583 if (ret) 584 return ret; 585 586 config_atu: 587 if (!(flags & PCI_BASE_ADDRESS_SPACE)) 588 type = PCIE_TLP_TYPE_MEM_RDWR; 589 else 590 type = PCIE_TLP_TYPE_IO_RDWR; 591 592 if (epf_bar->num_submap) 593 ret = dw_pcie_ep_ib_atu_addr(ep, func_no, type, epf_bar); 594 else 595 ret = dw_pcie_ep_ib_atu_bar(ep, func_no, type, 596 epf_bar->phys_addr, bar, size); 597 598 if (ret) 599 return ret; 600 601 ep_func->epf_bar[bar] = epf_bar; 602 603 return 0; 604 } 605 606 static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr, 607 u32 *atu_index) 608 { 609 u32 index; 610 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 611 612 for_each_set_bit(index, ep->ob_window_map, pci->num_ob_windows) { 613 if (ep->outbound_addr[index] != addr) 614 continue; 615 *atu_index = index; 616 return 0; 617 } 618 619 return -EINVAL; 620 } 621 622 static u64 dw_pcie_ep_align_addr(struct pci_epc *epc, u64 pci_addr, 623 size_t *pci_size, size_t *offset) 624 { 625 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 626 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 627 u64 mask = pci->region_align - 1; 628 size_t ofst = pci_addr & mask; 629 630 *pci_size = ALIGN(ofst + *pci_size, epc->mem->window.page_size); 631 *offset = ofst; 632 633 return pci_addr & ~mask; 634 } 635 636 static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 637 phys_addr_t addr) 638 { 639 int ret; 640 u32 atu_index; 641 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 642 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 643 644 ret = dw_pcie_find_index(ep, addr - pci->parent_bus_offset, 645 &atu_index); 646 if (ret < 0) 647 return; 648 649 ep->outbound_addr[atu_index] = 0; 650 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, atu_index); 651 clear_bit(atu_index, ep->ob_window_map); 652 } 653 654 static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 655 phys_addr_t addr, u64 pci_addr, size_t size) 656 { 657 int ret; 658 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 659 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 660 struct dw_pcie_ob_atu_cfg atu = { 0 }; 661 662 atu.func_no = func_no; 663 atu.type = PCIE_TLP_TYPE_MEM_RDWR; 664 atu.parent_bus_addr = addr - pci->parent_bus_offset; 665 atu.pci_addr = pci_addr; 666 atu.size = size; 667 ret = dw_pcie_ep_outbound_atu(ep, &atu); 668 if (ret) { 669 dev_err(pci->dev, "Failed to enable address\n"); 670 return ret; 671 } 672 673 return 0; 674 } 675 676 static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 677 { 678 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 679 struct dw_pcie_ep_func *ep_func; 680 u32 val, reg; 681 682 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 683 if (!ep_func || !ep_func->msi_cap) 684 return -EINVAL; 685 686 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 687 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 688 if (!(val & PCI_MSI_FLAGS_ENABLE)) 689 return -EINVAL; 690 691 val = FIELD_GET(PCI_MSI_FLAGS_QSIZE, val); 692 693 return 1 << val; 694 } 695 696 static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 697 u8 nr_irqs) 698 { 699 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 700 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 701 struct dw_pcie_ep_func *ep_func; 702 u8 mmc = order_base_2(nr_irqs); 703 u32 val, reg; 704 705 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 706 if (!ep_func || !ep_func->msi_cap) 707 return -EINVAL; 708 709 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 710 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 711 FIELD_MODIFY(PCI_MSI_FLAGS_QMASK, &val, mmc); 712 dw_pcie_dbi_ro_wr_en(pci); 713 dw_pcie_ep_writew_dbi(ep, func_no, reg, val); 714 dw_pcie_dbi_ro_wr_dis(pci); 715 716 return 0; 717 } 718 719 static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 720 { 721 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 722 struct dw_pcie_ep_func *ep_func; 723 u32 val, reg; 724 725 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 726 if (!ep_func || !ep_func->msix_cap) 727 return -EINVAL; 728 729 reg = ep_func->msix_cap + PCI_MSIX_FLAGS; 730 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 731 if (!(val & PCI_MSIX_FLAGS_ENABLE)) 732 return -EINVAL; 733 734 val &= PCI_MSIX_FLAGS_QSIZE; 735 736 return val + 1; 737 } 738 739 static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 740 u16 nr_irqs, enum pci_barno bir, u32 offset) 741 { 742 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 743 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 744 struct dw_pcie_ep_func *ep_func; 745 u32 val, reg; 746 747 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 748 if (!ep_func || !ep_func->msix_cap) 749 return -EINVAL; 750 751 dw_pcie_dbi_ro_wr_en(pci); 752 753 reg = ep_func->msix_cap + PCI_MSIX_FLAGS; 754 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 755 val &= ~PCI_MSIX_FLAGS_QSIZE; 756 val |= nr_irqs - 1; /* encoded as N-1 */ 757 dw_pcie_ep_writew_dbi(ep, func_no, reg, val); 758 759 reg = ep_func->msix_cap + PCI_MSIX_TABLE; 760 val = offset | bir; 761 dw_pcie_ep_writel_dbi(ep, func_no, reg, val); 762 763 reg = ep_func->msix_cap + PCI_MSIX_PBA; 764 val = (offset + (nr_irqs * PCI_MSIX_ENTRY_SIZE)) | bir; 765 dw_pcie_ep_writel_dbi(ep, func_no, reg, val); 766 767 dw_pcie_dbi_ro_wr_dis(pci); 768 769 return 0; 770 } 771 772 static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 773 unsigned int type, u16 interrupt_num) 774 { 775 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 776 777 if (!ep->ops->raise_irq) 778 return -EINVAL; 779 780 return ep->ops->raise_irq(ep, func_no, type, interrupt_num); 781 } 782 783 static void dw_pcie_ep_stop(struct pci_epc *epc) 784 { 785 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 786 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 787 788 /* 789 * Tear down the dedicated outbound window used for MSI 790 * generation. This avoids leaking an iATU window across 791 * endpoint stop/start cycles. 792 */ 793 if (ep->msi_iatu_mapped) { 794 dw_pcie_ep_unmap_addr(epc, 0, 0, ep->msi_mem_phys); 795 ep->msi_iatu_mapped = false; 796 } 797 798 dw_pcie_stop_link(pci); 799 } 800 801 static int dw_pcie_ep_start(struct pci_epc *epc) 802 { 803 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 804 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 805 806 return dw_pcie_start_link(pci); 807 } 808 809 static const struct pci_epc_features* 810 dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 811 { 812 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 813 814 if (!ep->ops->get_features) 815 return NULL; 816 817 return ep->ops->get_features(ep); 818 } 819 820 static const struct pci_epc_bar_rsvd_region * 821 dw_pcie_ep_find_bar_rsvd_region(struct dw_pcie_ep *ep, 822 enum pci_epc_bar_rsvd_region_type type, 823 enum pci_barno *bar, 824 resource_size_t *bar_offset) 825 { 826 const struct pci_epc_features *features; 827 const struct pci_epc_bar_desc *bar_desc; 828 const struct pci_epc_bar_rsvd_region *r; 829 int i, j; 830 831 if (!ep->ops->get_features) 832 return NULL; 833 834 features = ep->ops->get_features(ep); 835 if (!features) 836 return NULL; 837 838 for (i = BAR_0; i <= BAR_5; i++) { 839 bar_desc = &features->bar[i]; 840 841 if (!bar_desc->nr_rsvd_regions || !bar_desc->rsvd_regions) 842 continue; 843 844 for (j = 0; j < bar_desc->nr_rsvd_regions; j++) { 845 r = &bar_desc->rsvd_regions[j]; 846 847 if (r->type != type) 848 continue; 849 850 if (bar) 851 *bar = i; 852 if (bar_offset) 853 *bar_offset = r->offset; 854 return r; 855 } 856 } 857 858 return NULL; 859 } 860 861 static int 862 dw_pcie_ep_get_aux_resources_count(struct pci_epc *epc, u8 func_no, 863 u8 vfunc_no) 864 { 865 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 866 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 867 struct dw_edma_chip *edma = &pci->edma; 868 869 if (!pci->edma_reg_size) 870 return 0; 871 872 if (edma->db_offset == ~0) 873 return 0; 874 875 return 1; 876 } 877 878 static int 879 dw_pcie_ep_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 880 struct pci_epc_aux_resource *resources, 881 int num_resources) 882 { 883 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 884 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 885 const struct pci_epc_bar_rsvd_region *rsvd; 886 struct dw_edma_chip *edma = &pci->edma; 887 enum pci_barno dma_ctrl_bar = NO_BAR; 888 resource_size_t db_offset = edma->db_offset; 889 resource_size_t dma_ctrl_bar_offset = 0; 890 resource_size_t dma_reg_size; 891 int count; 892 893 count = dw_pcie_ep_get_aux_resources_count(epc, func_no, vfunc_no); 894 if (count < 0) 895 return count; 896 897 if (num_resources < count) 898 return -ENOSPC; 899 900 if (!count) 901 return 0; 902 903 dma_reg_size = pci->edma_reg_size; 904 905 rsvd = dw_pcie_ep_find_bar_rsvd_region(ep, 906 PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO, 907 &dma_ctrl_bar, 908 &dma_ctrl_bar_offset); 909 if (rsvd && rsvd->size < dma_reg_size) 910 dma_reg_size = rsvd->size; 911 912 /* 913 * For interrupt-emulation doorbells, report a standalone resource 914 * instead of bundling it into the DMA controller MMIO resource. 915 */ 916 if (range_end_overflows_t(resource_size_t, db_offset, 917 sizeof(u32), dma_reg_size)) 918 return -EINVAL; 919 920 resources[0] = (struct pci_epc_aux_resource) { 921 .type = PCI_EPC_AUX_DOORBELL_MMIO, 922 .phys_addr = pci->edma_reg_phys + db_offset, 923 .size = sizeof(u32), 924 .bar = dma_ctrl_bar, 925 .bar_offset = dma_ctrl_bar != NO_BAR ? 926 dma_ctrl_bar_offset + db_offset : 0, 927 .u.db_mmio = { 928 .irq = edma->db_irq, 929 .data = 0, /* write 0 to assert */ 930 }, 931 }; 932 933 return 0; 934 } 935 936 static const struct pci_epc_ops epc_ops = { 937 .write_header = dw_pcie_ep_write_header, 938 .set_bar = dw_pcie_ep_set_bar, 939 .clear_bar = dw_pcie_ep_clear_bar, 940 .align_addr = dw_pcie_ep_align_addr, 941 .map_addr = dw_pcie_ep_map_addr, 942 .unmap_addr = dw_pcie_ep_unmap_addr, 943 .set_msi = dw_pcie_ep_set_msi, 944 .get_msi = dw_pcie_ep_get_msi, 945 .set_msix = dw_pcie_ep_set_msix, 946 .get_msix = dw_pcie_ep_get_msix, 947 .raise_irq = dw_pcie_ep_raise_irq, 948 .start = dw_pcie_ep_start, 949 .stop = dw_pcie_ep_stop, 950 .get_features = dw_pcie_ep_get_features, 951 .get_aux_resources_count = dw_pcie_ep_get_aux_resources_count, 952 .get_aux_resources = dw_pcie_ep_get_aux_resources, 953 }; 954 955 /** 956 * dw_pcie_ep_raise_intx_irq - Raise INTx IRQ to the host 957 * @ep: DWC EP device 958 * @func_no: Function number of the endpoint 959 * 960 * Return: 0 if success, errno otherwise. 961 */ 962 int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) 963 { 964 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 965 struct device *dev = pci->dev; 966 967 dev_err(dev, "EP cannot raise INTX IRQs\n"); 968 969 return -EINVAL; 970 } 971 EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_intx_irq); 972 973 /** 974 * dw_pcie_ep_raise_msi_irq - Raise MSI IRQ to the host 975 * @ep: DWC EP device 976 * @func_no: Function number of the endpoint 977 * @interrupt_num: Interrupt number to be raised 978 * 979 * Return: 0 if success, errno otherwise. 980 */ 981 int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, 982 u8 interrupt_num) 983 { 984 u32 msg_addr_lower, msg_addr_upper, reg; 985 struct dw_pcie_ep_func *ep_func; 986 struct pci_epc *epc = ep->epc; 987 size_t map_size = sizeof(u32); 988 size_t offset; 989 u16 msg_ctrl, msg_data; 990 bool has_upper; 991 u64 msg_addr; 992 int ret; 993 994 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 995 if (!ep_func || !ep_func->msi_cap) 996 return -EINVAL; 997 998 /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ 999 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 1000 msg_ctrl = dw_pcie_ep_readw_dbi(ep, func_no, reg); 1001 has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); 1002 reg = ep_func->msi_cap + PCI_MSI_ADDRESS_LO; 1003 msg_addr_lower = dw_pcie_ep_readl_dbi(ep, func_no, reg); 1004 if (has_upper) { 1005 reg = ep_func->msi_cap + PCI_MSI_ADDRESS_HI; 1006 msg_addr_upper = dw_pcie_ep_readl_dbi(ep, func_no, reg); 1007 reg = ep_func->msi_cap + PCI_MSI_DATA_64; 1008 msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); 1009 } else { 1010 msg_addr_upper = 0; 1011 reg = ep_func->msi_cap + PCI_MSI_DATA_32; 1012 msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); 1013 } 1014 msg_addr = ((u64)msg_addr_upper) << 32 | msg_addr_lower; 1015 1016 msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); 1017 1018 /* 1019 * Program the outbound iATU once and keep it enabled. 1020 * 1021 * The spec warns that updating iATU registers while there are 1022 * operations in flight on the AXI bridge interface is not 1023 * supported, so we avoid reprogramming the region on every MSI, 1024 * specifically unmapping immediately after writel(). 1025 */ 1026 if (ep->msi_iatu_mapped && (ep->msi_msg_addr != msg_addr || 1027 ep->msi_map_size != map_size)) { 1028 /* 1029 * The host changed the MSI target address or the required 1030 * mapping size changed. Reprogramming the iATU when there are 1031 * operations in flight is unsafe on this controller. However, 1032 * there is no unified way to check if we have operations in 1033 * flight, thus we don't know if we should WARN() or not. 1034 */ 1035 /* flush posted write before unmap */ 1036 readl(ep->msi_mem + ep->msi_iatu_mapped_offset); 1037 dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); 1038 ep->msi_iatu_mapped = false; 1039 } 1040 1041 if (!ep->msi_iatu_mapped) { 1042 ret = dw_pcie_ep_map_addr(epc, func_no, 0, 1043 ep->msi_mem_phys, msg_addr, 1044 map_size); 1045 if (ret) 1046 return ret; 1047 1048 ep->msi_iatu_mapped = true; 1049 ep->msi_iatu_mapped_offset = offset; 1050 ep->msi_msg_addr = msg_addr; 1051 ep->msi_map_size = map_size; 1052 } 1053 1054 writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); 1055 1056 return 0; 1057 } 1058 EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msi_irq); 1059 1060 /** 1061 * dw_pcie_ep_raise_msix_irq_doorbell - Raise MSI-X to the host using Doorbell 1062 * method 1063 * @ep: DWC EP device 1064 * @func_no: Function number of the endpoint device 1065 * @interrupt_num: Interrupt number to be raised 1066 * 1067 * Return: 0 if success, errno otherwise. 1068 */ 1069 int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, 1070 u16 interrupt_num) 1071 { 1072 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1073 struct dw_pcie_ep_func *ep_func; 1074 u32 msg_data; 1075 1076 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 1077 if (!ep_func || !ep_func->msix_cap) 1078 return -EINVAL; 1079 1080 msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | 1081 (interrupt_num - 1); 1082 1083 dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); 1084 1085 return 0; 1086 } 1087 1088 /** 1089 * dw_pcie_ep_raise_msix_irq - Raise MSI-X to the host 1090 * @ep: DWC EP device 1091 * @func_no: Function number of the endpoint device 1092 * @interrupt_num: Interrupt number to be raised 1093 * 1094 * Return: 0 if success, errno otherwise. 1095 */ 1096 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, 1097 u16 interrupt_num) 1098 { 1099 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1100 struct pci_epf_msix_tbl *msix_tbl; 1101 struct dw_pcie_ep_func *ep_func; 1102 struct pci_epc *epc = ep->epc; 1103 size_t map_size = sizeof(u32); 1104 size_t offset; 1105 u32 reg, msg_data, vec_ctrl; 1106 u32 tbl_offset; 1107 u64 msg_addr; 1108 int ret; 1109 u8 bir; 1110 1111 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 1112 if (!ep_func || !ep_func->msix_cap) 1113 return -EINVAL; 1114 1115 reg = ep_func->msix_cap + PCI_MSIX_TABLE; 1116 tbl_offset = dw_pcie_ep_readl_dbi(ep, func_no, reg); 1117 bir = FIELD_GET(PCI_MSIX_TABLE_BIR, tbl_offset); 1118 tbl_offset &= PCI_MSIX_TABLE_OFFSET; 1119 1120 msix_tbl = ep_func->epf_bar[bir]->addr + tbl_offset; 1121 msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; 1122 msg_data = msix_tbl[(interrupt_num - 1)].msg_data; 1123 vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl; 1124 1125 if (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT) { 1126 dev_dbg(pci->dev, "MSI-X entry ctrl set\n"); 1127 return -EPERM; 1128 } 1129 1130 /* 1131 * ep->msi_iatu_mapped means that an MSI target address is cached, 1132 * unmap it first so that we can reuse ep->msi_mem_phys for MSI-X. 1133 */ 1134 if (ep->msi_iatu_mapped) { 1135 /* flush posted write before unmap */ 1136 readl(ep->msi_mem + ep->msi_iatu_mapped_offset); 1137 dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); 1138 ep->msi_iatu_mapped = false; 1139 } 1140 1141 msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); 1142 ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, 1143 map_size); 1144 if (ret) 1145 return ret; 1146 1147 writel(msg_data, ep->msi_mem + offset); 1148 1149 /* flush posted write before unmap */ 1150 readl(ep->msi_mem + offset); 1151 1152 dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); 1153 1154 return 0; 1155 } 1156 EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msix_irq); 1157 1158 /** 1159 * dw_pcie_ep_cleanup - Cleanup DWC EP resources after fundamental reset 1160 * @ep: DWC EP device 1161 * 1162 * Cleans up the DWC EP specific resources like eDMA etc... after fundamental 1163 * reset like PERST#. Note that this API is only applicable for drivers 1164 * supporting PERST# or any other methods of fundamental reset. 1165 */ 1166 void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) 1167 { 1168 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1169 1170 if (ep->msi_iatu_mapped) { 1171 dw_pcie_ep_unmap_addr(ep->epc, 0, 0, ep->msi_mem_phys); 1172 ep->msi_iatu_mapped = false; 1173 } 1174 1175 dwc_pcie_debugfs_deinit(pci); 1176 dw_pcie_edma_remove(pci); 1177 } 1178 EXPORT_SYMBOL_GPL(dw_pcie_ep_cleanup); 1179 1180 /** 1181 * dw_pcie_ep_deinit - Deinitialize the endpoint device 1182 * @ep: DWC EP device 1183 * 1184 * Deinitialize the endpoint device. EPC device is not destroyed since that will 1185 * be taken care by Devres. 1186 */ 1187 void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) 1188 { 1189 struct pci_epc *epc = ep->epc; 1190 1191 dw_pcie_ep_cleanup(ep); 1192 1193 pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, 1194 epc->mem->window.page_size); 1195 1196 pci_epc_mem_exit(epc); 1197 } 1198 EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); 1199 1200 static void dw_pcie_ep_init_rebar_registers(struct dw_pcie_ep *ep, u8 func_no) 1201 { 1202 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 1203 unsigned int offset, nbars; 1204 enum pci_barno bar; 1205 u32 reg, i, val; 1206 1207 if (!ep_func) 1208 return; 1209 1210 offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); 1211 1212 if (offset) { 1213 reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 1214 nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); 1215 1216 /* 1217 * PCIe r6.0, sec 7.8.6.2 require us to support at least one 1218 * size in the range from 1 MB to 512 GB. Advertise support 1219 * for 1 MB BAR size only. 1220 * 1221 * For a BAR that has been configured via dw_pcie_ep_set_bar(), 1222 * advertise support for only that size instead. 1223 */ 1224 for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { 1225 /* 1226 * While the RESBAR_CAP_REG_* fields are sticky, the 1227 * RESBAR_CTRL_REG_BAR_SIZE field is non-sticky (it is 1228 * sticky in certain versions of DWC PCIe, but not all). 1229 * 1230 * RESBAR_CTRL_REG_BAR_SIZE is updated automatically by 1231 * the controller when RESBAR_CAP_REG is written, which 1232 * is why RESBAR_CAP_REG is written here. 1233 */ 1234 val = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 1235 bar = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, val); 1236 if (ep_func->epf_bar[bar]) 1237 pci_epc_bar_size_to_rebar_cap(ep_func->epf_bar[bar]->size, &val); 1238 else 1239 val = BIT(4); 1240 1241 dw_pcie_ep_writel_dbi(ep, func_no, offset + PCI_REBAR_CAP, val); 1242 } 1243 } 1244 } 1245 1246 static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) 1247 { 1248 struct dw_pcie_ep *ep = &pci->ep; 1249 u8 funcs = ep->epc->max_functions; 1250 u32 func0_lnkcap, lnkcap; 1251 u8 func_no, offset; 1252 1253 dw_pcie_dbi_ro_wr_en(pci); 1254 1255 for (func_no = 0; func_no < funcs; func_no++) 1256 dw_pcie_ep_init_rebar_registers(ep, func_no); 1257 1258 dw_pcie_setup(pci); 1259 1260 /* 1261 * PCIe r7.0, section 7.5.3.6 states that for multi-function 1262 * endpoints, max link width and speed fields must report same 1263 * values for all functions. However, dw_pcie_setup() programs 1264 * these fields only for function 0. Hence, mirror these fields 1265 * to all other functions as well. 1266 */ 1267 if (funcs > 1) { 1268 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); 1269 func0_lnkcap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); 1270 func0_lnkcap = FIELD_GET(PCI_EXP_LNKCAP_MLW | 1271 PCI_EXP_LNKCAP_SLS, func0_lnkcap); 1272 1273 for (func_no = 1; func_no < funcs; func_no++) { 1274 offset = dw_pcie_ep_find_capability(ep, func_no, 1275 PCI_CAP_ID_EXP); 1276 lnkcap = dw_pcie_ep_readl_dbi(ep, func_no, 1277 offset + PCI_EXP_LNKCAP); 1278 FIELD_MODIFY(PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS, 1279 &lnkcap, func0_lnkcap); 1280 dw_pcie_ep_writel_dbi(ep, func_no, 1281 offset + PCI_EXP_LNKCAP, lnkcap); 1282 } 1283 } 1284 1285 dw_pcie_dbi_ro_wr_dis(pci); 1286 } 1287 1288 static void dw_pcie_ep_disable_bars(struct dw_pcie_ep *ep) 1289 { 1290 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1291 enum pci_epc_bar_type bar_type; 1292 enum pci_barno bar; 1293 1294 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 1295 bar_type = dw_pcie_ep_get_bar_type(ep, bar); 1296 1297 /* 1298 * Reserved BARs should not get disabled by default. All other 1299 * BAR types are disabled by default. 1300 * 1301 * This is in line with the current EPC core design, where all 1302 * BARs are disabled by default, and then the EPF driver enables 1303 * the BARs it wishes to use. 1304 */ 1305 if (bar_type != BAR_RESERVED) 1306 dw_pcie_ep_reset_bar(pci, bar); 1307 } 1308 } 1309 1310 /** 1311 * dw_pcie_ep_init_registers - Initialize DWC EP specific registers 1312 * @ep: DWC EP device 1313 * 1314 * Initialize the registers (CSRs) specific to DWC EP. This API should be called 1315 * only when the endpoint receives an active refclk (either from host or 1316 * generated locally). 1317 */ 1318 int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) 1319 { 1320 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1321 struct dw_pcie_ep_func *ep_func; 1322 struct device *dev = pci->dev; 1323 struct pci_epc *epc = ep->epc; 1324 u32 ptm_cap_base, reg; 1325 u8 hdr_type; 1326 u8 func_no; 1327 void *addr; 1328 int ret; 1329 1330 hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE) & 1331 PCI_HEADER_TYPE_MASK; 1332 if (hdr_type != PCI_HEADER_TYPE_NORMAL) { 1333 dev_err(pci->dev, 1334 "PCIe controller is not set to EP mode (hdr_type:0x%x)!\n", 1335 hdr_type); 1336 return -EIO; 1337 } 1338 1339 dw_pcie_version_detect(pci); 1340 1341 dw_pcie_iatu_detect(pci); 1342 1343 ret = dw_pcie_edma_detect(pci); 1344 if (ret) 1345 return ret; 1346 1347 ret = -ENOMEM; 1348 if (!ep->ib_window_map) { 1349 ep->ib_window_map = devm_bitmap_zalloc(dev, pci->num_ib_windows, 1350 GFP_KERNEL); 1351 if (!ep->ib_window_map) 1352 goto err_remove_edma; 1353 } 1354 1355 if (!ep->ob_window_map) { 1356 ep->ob_window_map = devm_bitmap_zalloc(dev, pci->num_ob_windows, 1357 GFP_KERNEL); 1358 if (!ep->ob_window_map) 1359 goto err_remove_edma; 1360 } 1361 1362 if (!ep->outbound_addr) { 1363 addr = devm_kcalloc(dev, pci->num_ob_windows, sizeof(phys_addr_t), 1364 GFP_KERNEL); 1365 if (!addr) 1366 goto err_remove_edma; 1367 ep->outbound_addr = addr; 1368 } 1369 1370 for (func_no = 0; func_no < epc->max_functions; func_no++) { 1371 1372 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 1373 if (ep_func) 1374 continue; 1375 1376 ep_func = devm_kzalloc(dev, sizeof(*ep_func), GFP_KERNEL); 1377 if (!ep_func) 1378 goto err_remove_edma; 1379 1380 ep_func->func_no = func_no; 1381 ep_func->msi_cap = dw_pcie_ep_find_capability(ep, func_no, 1382 PCI_CAP_ID_MSI); 1383 ep_func->msix_cap = dw_pcie_ep_find_capability(ep, func_no, 1384 PCI_CAP_ID_MSIX); 1385 1386 list_add_tail(&ep_func->list, &ep->func_list); 1387 } 1388 1389 if (ep->ops->init) { 1390 ret = ep->ops->init(ep); 1391 if (ret) 1392 goto err_remove_edma; 1393 } 1394 1395 dw_pcie_ep_disable_bars(ep); 1396 1397 /* 1398 * PCIe r6.0, section 7.9.15 states that for endpoints that support 1399 * PTM, this capability structure is required in exactly one 1400 * function, which controls the PTM behavior of all PTM capable 1401 * functions. This indicates the PTM capability structure 1402 * represents controller-level registers rather than per-function 1403 * registers. 1404 * 1405 * Therefore, PTM capability registers are configured using the 1406 * standard DBI accessors, instead of func_no indexed per-function 1407 * accessors. 1408 */ 1409 ptm_cap_base = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_PTM); 1410 1411 /* 1412 * PTM responder capability can be disabled only after disabling 1413 * PTM root capability. 1414 */ 1415 if (ptm_cap_base) { 1416 dw_pcie_dbi_ro_wr_en(pci); 1417 reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); 1418 reg &= ~PCI_PTM_CAP_ROOT; 1419 dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); 1420 1421 reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); 1422 reg &= ~(PCI_PTM_CAP_RES | PCI_PTM_GRANULARITY_MASK); 1423 dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); 1424 dw_pcie_dbi_ro_wr_dis(pci); 1425 } 1426 1427 dw_pcie_ep_init_non_sticky_registers(pci); 1428 1429 dwc_pcie_debugfs_init(pci, DW_PCIE_EP_TYPE); 1430 1431 return 0; 1432 1433 err_remove_edma: 1434 dw_pcie_edma_remove(pci); 1435 1436 return ret; 1437 } 1438 EXPORT_SYMBOL_GPL(dw_pcie_ep_init_registers); 1439 1440 /** 1441 * dw_pcie_ep_linkup - Notify EPF drivers about Link Up event 1442 * @ep: DWC EP device 1443 */ 1444 void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) 1445 { 1446 struct pci_epc *epc = ep->epc; 1447 1448 pci_epc_linkup(epc); 1449 } 1450 EXPORT_SYMBOL_GPL(dw_pcie_ep_linkup); 1451 1452 /** 1453 * dw_pcie_ep_linkdown - Notify EPF drivers about Link Down event 1454 * @ep: DWC EP device 1455 * 1456 * Non-sticky registers are also initialized before sending the notification to 1457 * the EPF drivers. This is needed since the registers need to be initialized 1458 * before the link comes back again. 1459 */ 1460 void dw_pcie_ep_linkdown(struct dw_pcie_ep *ep) 1461 { 1462 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1463 struct pci_epc *epc = ep->epc; 1464 1465 /* 1466 * Initialize the non-sticky DWC registers as they would've reset post 1467 * Link Down. This is specifically needed for drivers not supporting 1468 * PERST# as they have no way to reinitialize the registers before the 1469 * link comes back again. 1470 */ 1471 dw_pcie_ep_init_non_sticky_registers(pci); 1472 1473 pci_epc_linkdown(epc); 1474 } 1475 EXPORT_SYMBOL_GPL(dw_pcie_ep_linkdown); 1476 1477 static int dw_pcie_ep_get_resources(struct dw_pcie_ep *ep) 1478 { 1479 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1480 struct device *dev = pci->dev; 1481 struct platform_device *pdev = to_platform_device(dev); 1482 struct device_node *np = dev->of_node; 1483 struct pci_epc *epc = ep->epc; 1484 struct resource *res; 1485 int ret; 1486 1487 ret = dw_pcie_get_resources(pci); 1488 if (ret) 1489 return ret; 1490 1491 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); 1492 if (!res) 1493 return -EINVAL; 1494 1495 ep->phys_base = res->start; 1496 ep->addr_size = resource_size(res); 1497 1498 /* 1499 * artpec6_pcie_cpu_addr_fixup() uses ep->phys_base, so call 1500 * dw_pcie_parent_bus_offset() after setting ep->phys_base. 1501 */ 1502 pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "addr_space", 1503 ep->phys_base); 1504 1505 ret = of_property_read_u8(np, "max-functions", &epc->max_functions); 1506 if (ret < 0) 1507 epc->max_functions = 1; 1508 1509 return 0; 1510 } 1511 1512 /** 1513 * dw_pcie_ep_init - Initialize the endpoint device 1514 * @ep: DWC EP device 1515 * 1516 * Initialize the endpoint device. Allocate resources and create the EPC 1517 * device with the endpoint framework. 1518 * 1519 * Return: 0 if success, errno otherwise. 1520 */ 1521 int dw_pcie_ep_init(struct dw_pcie_ep *ep) 1522 { 1523 int ret; 1524 struct pci_epc *epc; 1525 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1526 struct device *dev = pci->dev; 1527 1528 INIT_LIST_HEAD(&ep->func_list); 1529 ep->msi_iatu_mapped = false; 1530 ep->msi_msg_addr = 0; 1531 ep->msi_map_size = 0; 1532 1533 epc = devm_pci_epc_create(dev, &epc_ops); 1534 if (IS_ERR(epc)) { 1535 dev_err(dev, "Failed to create epc device\n"); 1536 return PTR_ERR(epc); 1537 } 1538 1539 ep->epc = epc; 1540 epc_set_drvdata(epc, ep); 1541 1542 ret = dw_pcie_ep_get_resources(ep); 1543 if (ret) 1544 return ret; 1545 1546 if (ep->ops->pre_init) { 1547 ret = ep->ops->pre_init(ep); 1548 if (ret) 1549 return ret; 1550 } 1551 1552 ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size, 1553 ep->page_size); 1554 if (ret < 0) { 1555 dev_err(dev, "Failed to initialize address space\n"); 1556 return ret; 1557 } 1558 1559 ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys, 1560 epc->mem->window.page_size); 1561 if (!ep->msi_mem) { 1562 ret = -ENOMEM; 1563 dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); 1564 goto err_exit_epc_mem; 1565 } 1566 1567 return 0; 1568 1569 err_exit_epc_mem: 1570 pci_epc_mem_exit(epc); 1571 1572 return ret; 1573 } 1574 EXPORT_SYMBOL_GPL(dw_pcie_ep_init); 1575