1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe Endpoint controller driver 4 * 5 * Copyright (C) 2017 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 #include <linux/align.h> 10 #include <linux/bitfield.h> 11 #include <linux/of.h> 12 #include <linux/platform_device.h> 13 14 #include "pcie-designware.h" 15 #include <linux/pci-epc.h> 16 #include <linux/pci-epf.h> 17 18 /** 19 * dw_pcie_ep_get_func_from_ep - Get the struct dw_pcie_ep_func corresponding to 20 * the endpoint function 21 * @ep: DWC EP device 22 * @func_no: Function number of the endpoint device 23 * 24 * Return: struct dw_pcie_ep_func if success, NULL otherwise. 25 */ 26 struct dw_pcie_ep_func * 27 dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) 28 { 29 struct dw_pcie_ep_func *ep_func; 30 31 list_for_each_entry(ep_func, &ep->func_list, list) { 32 if (ep_func->func_no == func_no) 33 return ep_func; 34 } 35 36 return NULL; 37 } 38 39 static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, 40 enum pci_barno bar, int flags) 41 { 42 struct dw_pcie_ep *ep = &pci->ep; 43 u32 reg; 44 45 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 46 dw_pcie_dbi_ro_wr_en(pci); 47 dw_pcie_ep_writel_dbi2(ep, func_no, reg, 0x0); 48 dw_pcie_ep_writel_dbi(ep, func_no, reg, 0x0); 49 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 50 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0x0); 51 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0x0); 52 } 53 dw_pcie_dbi_ro_wr_dis(pci); 54 } 55 56 /** 57 * dw_pcie_ep_reset_bar - Reset endpoint BAR 58 * @pci: DWC PCI device 59 * @bar: BAR number of the endpoint 60 */ 61 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) 62 { 63 u8 func_no, funcs; 64 65 funcs = pci->ep.epc->max_functions; 66 67 for (func_no = 0; func_no < funcs; func_no++) 68 __dw_pcie_ep_reset_bar(pci, func_no, bar, 0); 69 } 70 EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); 71 72 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie_ep *ep, u8 func_no, 73 u8 cap_ptr, u8 cap) 74 { 75 u8 cap_id, next_cap_ptr; 76 u16 reg; 77 78 if (!cap_ptr) 79 return 0; 80 81 reg = dw_pcie_ep_readw_dbi(ep, func_no, cap_ptr); 82 cap_id = (reg & 0x00ff); 83 84 if (cap_id > PCI_CAP_ID_MAX) 85 return 0; 86 87 if (cap_id == cap) 88 return cap_ptr; 89 90 next_cap_ptr = (reg & 0xff00) >> 8; 91 return __dw_pcie_ep_find_next_cap(ep, func_no, next_cap_ptr, cap); 92 } 93 94 static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) 95 { 96 u8 next_cap_ptr; 97 u16 reg; 98 99 reg = dw_pcie_ep_readw_dbi(ep, func_no, PCI_CAPABILITY_LIST); 100 next_cap_ptr = (reg & 0x00ff); 101 102 return __dw_pcie_ep_find_next_cap(ep, func_no, next_cap_ptr, cap); 103 } 104 105 static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 106 struct pci_epf_header *hdr) 107 { 108 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 109 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 110 111 dw_pcie_dbi_ro_wr_en(pci); 112 dw_pcie_ep_writew_dbi(ep, func_no, PCI_VENDOR_ID, hdr->vendorid); 113 dw_pcie_ep_writew_dbi(ep, func_no, PCI_DEVICE_ID, hdr->deviceid); 114 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_REVISION_ID, hdr->revid); 115 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CLASS_PROG, hdr->progif_code); 116 dw_pcie_ep_writew_dbi(ep, func_no, PCI_CLASS_DEVICE, 117 hdr->subclass_code | hdr->baseclass_code << 8); 118 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CACHE_LINE_SIZE, 119 hdr->cache_line_size); 120 dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_VENDOR_ID, 121 hdr->subsys_vendor_id); 122 dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_ID, hdr->subsys_id); 123 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_INTERRUPT_PIN, 124 hdr->interrupt_pin); 125 dw_pcie_dbi_ro_wr_dis(pci); 126 127 return 0; 128 } 129 130 static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, int type, 131 dma_addr_t cpu_addr, enum pci_barno bar, 132 size_t size) 133 { 134 int ret; 135 u32 free_win; 136 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 137 138 if (!ep->bar_to_atu[bar]) 139 free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); 140 else 141 free_win = ep->bar_to_atu[bar] - 1; 142 143 if (free_win >= pci->num_ib_windows) { 144 dev_err(pci->dev, "No free inbound window\n"); 145 return -EINVAL; 146 } 147 148 ret = dw_pcie_prog_ep_inbound_atu(pci, func_no, free_win, type, 149 cpu_addr, bar, size); 150 if (ret < 0) { 151 dev_err(pci->dev, "Failed to program IB window\n"); 152 return ret; 153 } 154 155 /* 156 * Always increment free_win before assignment, since value 0 is used to identify 157 * unallocated mapping. 158 */ 159 ep->bar_to_atu[bar] = free_win + 1; 160 set_bit(free_win, ep->ib_window_map); 161 162 return 0; 163 } 164 165 static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, 166 struct dw_pcie_ob_atu_cfg *atu) 167 { 168 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 169 u32 free_win; 170 int ret; 171 172 free_win = find_first_zero_bit(ep->ob_window_map, pci->num_ob_windows); 173 if (free_win >= pci->num_ob_windows) { 174 dev_err(pci->dev, "No free outbound window\n"); 175 return -EINVAL; 176 } 177 178 atu->index = free_win; 179 ret = dw_pcie_prog_outbound_atu(pci, atu); 180 if (ret) 181 return ret; 182 183 set_bit(free_win, ep->ob_window_map); 184 ep->outbound_addr[free_win] = atu->cpu_addr; 185 186 return 0; 187 } 188 189 static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 190 struct pci_epf_bar *epf_bar) 191 { 192 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 193 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 194 enum pci_barno bar = epf_bar->barno; 195 u32 atu_index = ep->bar_to_atu[bar] - 1; 196 197 if (!ep->bar_to_atu[bar]) 198 return; 199 200 __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); 201 202 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); 203 clear_bit(atu_index, ep->ib_window_map); 204 ep->epf_bar[bar] = NULL; 205 ep->bar_to_atu[bar] = 0; 206 } 207 208 static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 209 struct pci_epf_bar *epf_bar) 210 { 211 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 212 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 213 enum pci_barno bar = epf_bar->barno; 214 size_t size = epf_bar->size; 215 int flags = epf_bar->flags; 216 int ret, type; 217 u32 reg; 218 219 /* 220 * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs 221 * 1 and 2 to form a 64-bit BAR. 222 */ 223 if ((flags & PCI_BASE_ADDRESS_MEM_TYPE_64) && (bar & 1)) 224 return -EINVAL; 225 226 /* 227 * Certain EPF drivers dynamically change the physical address of a BAR 228 * (i.e. they call set_bar() twice, without ever calling clear_bar(), as 229 * calling clear_bar() would clear the BAR's PCI address assigned by the 230 * host). 231 */ 232 if (ep->epf_bar[bar]) { 233 /* 234 * We can only dynamically change a BAR if the new BAR size and 235 * BAR flags do not differ from the existing configuration. 236 */ 237 if (ep->epf_bar[bar]->barno != bar || 238 ep->epf_bar[bar]->size != size || 239 ep->epf_bar[bar]->flags != flags) 240 return -EINVAL; 241 242 /* 243 * When dynamically changing a BAR, skip writing the BAR reg, as 244 * that would clear the BAR's PCI address assigned by the host. 245 */ 246 goto config_atu; 247 } 248 249 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 250 251 dw_pcie_dbi_ro_wr_en(pci); 252 253 dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1)); 254 dw_pcie_ep_writel_dbi(ep, func_no, reg, flags); 255 256 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 257 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1)); 258 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0); 259 } 260 261 dw_pcie_dbi_ro_wr_dis(pci); 262 263 config_atu: 264 if (!(flags & PCI_BASE_ADDRESS_SPACE)) 265 type = PCIE_ATU_TYPE_MEM; 266 else 267 type = PCIE_ATU_TYPE_IO; 268 269 ret = dw_pcie_ep_inbound_atu(ep, func_no, type, epf_bar->phys_addr, bar, 270 size); 271 if (ret) 272 return ret; 273 274 ep->epf_bar[bar] = epf_bar; 275 276 return 0; 277 } 278 279 static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr, 280 u32 *atu_index) 281 { 282 u32 index; 283 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 284 285 for (index = 0; index < pci->num_ob_windows; index++) { 286 if (ep->outbound_addr[index] != addr) 287 continue; 288 *atu_index = index; 289 return 0; 290 } 291 292 return -EINVAL; 293 } 294 295 static u64 dw_pcie_ep_align_addr(struct pci_epc *epc, u64 pci_addr, 296 size_t *pci_size, size_t *offset) 297 { 298 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 299 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 300 u64 mask = pci->region_align - 1; 301 size_t ofst = pci_addr & mask; 302 303 *pci_size = ALIGN(ofst + *pci_size, epc->mem->window.page_size); 304 *offset = ofst; 305 306 return pci_addr & ~mask; 307 } 308 309 static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 310 phys_addr_t addr) 311 { 312 int ret; 313 u32 atu_index; 314 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 315 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 316 317 ret = dw_pcie_find_index(ep, addr, &atu_index); 318 if (ret < 0) 319 return; 320 321 ep->outbound_addr[atu_index] = 0; 322 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, atu_index); 323 clear_bit(atu_index, ep->ob_window_map); 324 } 325 326 static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 327 phys_addr_t addr, u64 pci_addr, size_t size) 328 { 329 int ret; 330 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 331 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 332 struct dw_pcie_ob_atu_cfg atu = { 0 }; 333 334 atu.func_no = func_no; 335 atu.type = PCIE_ATU_TYPE_MEM; 336 atu.cpu_addr = addr; 337 atu.pci_addr = pci_addr; 338 atu.size = size; 339 ret = dw_pcie_ep_outbound_atu(ep, &atu); 340 if (ret) { 341 dev_err(pci->dev, "Failed to enable address\n"); 342 return ret; 343 } 344 345 return 0; 346 } 347 348 static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 349 { 350 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 351 struct dw_pcie_ep_func *ep_func; 352 u32 val, reg; 353 354 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 355 if (!ep_func || !ep_func->msi_cap) 356 return -EINVAL; 357 358 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 359 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 360 if (!(val & PCI_MSI_FLAGS_ENABLE)) 361 return -EINVAL; 362 363 val = FIELD_GET(PCI_MSI_FLAGS_QSIZE, val); 364 365 return val; 366 } 367 368 static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 369 u8 interrupts) 370 { 371 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 372 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 373 struct dw_pcie_ep_func *ep_func; 374 u32 val, reg; 375 376 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 377 if (!ep_func || !ep_func->msi_cap) 378 return -EINVAL; 379 380 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 381 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 382 val &= ~PCI_MSI_FLAGS_QMASK; 383 val |= FIELD_PREP(PCI_MSI_FLAGS_QMASK, interrupts); 384 dw_pcie_dbi_ro_wr_en(pci); 385 dw_pcie_ep_writew_dbi(ep, func_no, reg, val); 386 dw_pcie_dbi_ro_wr_dis(pci); 387 388 return 0; 389 } 390 391 static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 392 { 393 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 394 struct dw_pcie_ep_func *ep_func; 395 u32 val, reg; 396 397 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 398 if (!ep_func || !ep_func->msix_cap) 399 return -EINVAL; 400 401 reg = ep_func->msix_cap + PCI_MSIX_FLAGS; 402 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 403 if (!(val & PCI_MSIX_FLAGS_ENABLE)) 404 return -EINVAL; 405 406 val &= PCI_MSIX_FLAGS_QSIZE; 407 408 return val; 409 } 410 411 static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 412 u16 interrupts, enum pci_barno bir, u32 offset) 413 { 414 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 415 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 416 struct dw_pcie_ep_func *ep_func; 417 u32 val, reg; 418 419 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 420 if (!ep_func || !ep_func->msix_cap) 421 return -EINVAL; 422 423 dw_pcie_dbi_ro_wr_en(pci); 424 425 reg = ep_func->msix_cap + PCI_MSIX_FLAGS; 426 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 427 val &= ~PCI_MSIX_FLAGS_QSIZE; 428 val |= interrupts; 429 dw_pcie_writew_dbi(pci, reg, val); 430 431 reg = ep_func->msix_cap + PCI_MSIX_TABLE; 432 val = offset | bir; 433 dw_pcie_ep_writel_dbi(ep, func_no, reg, val); 434 435 reg = ep_func->msix_cap + PCI_MSIX_PBA; 436 val = (offset + (interrupts * PCI_MSIX_ENTRY_SIZE)) | bir; 437 dw_pcie_ep_writel_dbi(ep, func_no, reg, val); 438 439 dw_pcie_dbi_ro_wr_dis(pci); 440 441 return 0; 442 } 443 444 static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 445 unsigned int type, u16 interrupt_num) 446 { 447 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 448 449 if (!ep->ops->raise_irq) 450 return -EINVAL; 451 452 return ep->ops->raise_irq(ep, func_no, type, interrupt_num); 453 } 454 455 static void dw_pcie_ep_stop(struct pci_epc *epc) 456 { 457 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 458 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 459 460 dw_pcie_stop_link(pci); 461 } 462 463 static int dw_pcie_ep_start(struct pci_epc *epc) 464 { 465 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 466 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 467 468 return dw_pcie_start_link(pci); 469 } 470 471 static const struct pci_epc_features* 472 dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 473 { 474 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 475 476 if (!ep->ops->get_features) 477 return NULL; 478 479 return ep->ops->get_features(ep); 480 } 481 482 static const struct pci_epc_ops epc_ops = { 483 .write_header = dw_pcie_ep_write_header, 484 .set_bar = dw_pcie_ep_set_bar, 485 .clear_bar = dw_pcie_ep_clear_bar, 486 .align_addr = dw_pcie_ep_align_addr, 487 .map_addr = dw_pcie_ep_map_addr, 488 .unmap_addr = dw_pcie_ep_unmap_addr, 489 .set_msi = dw_pcie_ep_set_msi, 490 .get_msi = dw_pcie_ep_get_msi, 491 .set_msix = dw_pcie_ep_set_msix, 492 .get_msix = dw_pcie_ep_get_msix, 493 .raise_irq = dw_pcie_ep_raise_irq, 494 .start = dw_pcie_ep_start, 495 .stop = dw_pcie_ep_stop, 496 .get_features = dw_pcie_ep_get_features, 497 }; 498 499 /** 500 * dw_pcie_ep_raise_intx_irq - Raise INTx IRQ to the host 501 * @ep: DWC EP device 502 * @func_no: Function number of the endpoint 503 * 504 * Return: 0 if success, errono otherwise. 505 */ 506 int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) 507 { 508 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 509 struct device *dev = pci->dev; 510 511 dev_err(dev, "EP cannot raise INTX IRQs\n"); 512 513 return -EINVAL; 514 } 515 EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_intx_irq); 516 517 /** 518 * dw_pcie_ep_raise_msi_irq - Raise MSI IRQ to the host 519 * @ep: DWC EP device 520 * @func_no: Function number of the endpoint 521 * @interrupt_num: Interrupt number to be raised 522 * 523 * Return: 0 if success, errono otherwise. 524 */ 525 int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, 526 u8 interrupt_num) 527 { 528 u32 msg_addr_lower, msg_addr_upper, reg; 529 struct dw_pcie_ep_func *ep_func; 530 struct pci_epc *epc = ep->epc; 531 size_t map_size = sizeof(u32); 532 size_t offset; 533 u16 msg_ctrl, msg_data; 534 bool has_upper; 535 u64 msg_addr; 536 int ret; 537 538 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 539 if (!ep_func || !ep_func->msi_cap) 540 return -EINVAL; 541 542 /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ 543 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 544 msg_ctrl = dw_pcie_ep_readw_dbi(ep, func_no, reg); 545 has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); 546 reg = ep_func->msi_cap + PCI_MSI_ADDRESS_LO; 547 msg_addr_lower = dw_pcie_ep_readl_dbi(ep, func_no, reg); 548 if (has_upper) { 549 reg = ep_func->msi_cap + PCI_MSI_ADDRESS_HI; 550 msg_addr_upper = dw_pcie_ep_readl_dbi(ep, func_no, reg); 551 reg = ep_func->msi_cap + PCI_MSI_DATA_64; 552 msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); 553 } else { 554 msg_addr_upper = 0; 555 reg = ep_func->msi_cap + PCI_MSI_DATA_32; 556 msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); 557 } 558 msg_addr = ((u64)msg_addr_upper) << 32 | msg_addr_lower; 559 560 msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); 561 ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, 562 map_size); 563 if (ret) 564 return ret; 565 566 writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); 567 568 dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); 569 570 return 0; 571 } 572 EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msi_irq); 573 574 /** 575 * dw_pcie_ep_raise_msix_irq_doorbell - Raise MSI-X to the host using Doorbell 576 * method 577 * @ep: DWC EP device 578 * @func_no: Function number of the endpoint device 579 * @interrupt_num: Interrupt number to be raised 580 * 581 * Return: 0 if success, errno otherwise. 582 */ 583 int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, 584 u16 interrupt_num) 585 { 586 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 587 struct dw_pcie_ep_func *ep_func; 588 u32 msg_data; 589 590 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 591 if (!ep_func || !ep_func->msix_cap) 592 return -EINVAL; 593 594 msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | 595 (interrupt_num - 1); 596 597 dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); 598 599 return 0; 600 } 601 602 /** 603 * dw_pcie_ep_raise_msix_irq - Raise MSI-X to the host 604 * @ep: DWC EP device 605 * @func_no: Function number of the endpoint device 606 * @interrupt_num: Interrupt number to be raised 607 * 608 * Return: 0 if success, errno otherwise. 609 */ 610 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, 611 u16 interrupt_num) 612 { 613 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 614 struct pci_epf_msix_tbl *msix_tbl; 615 struct dw_pcie_ep_func *ep_func; 616 struct pci_epc *epc = ep->epc; 617 size_t map_size = sizeof(u32); 618 size_t offset; 619 u32 reg, msg_data, vec_ctrl; 620 u32 tbl_offset; 621 u64 msg_addr; 622 int ret; 623 u8 bir; 624 625 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 626 if (!ep_func || !ep_func->msix_cap) 627 return -EINVAL; 628 629 reg = ep_func->msix_cap + PCI_MSIX_TABLE; 630 tbl_offset = dw_pcie_ep_readl_dbi(ep, func_no, reg); 631 bir = FIELD_GET(PCI_MSIX_TABLE_BIR, tbl_offset); 632 tbl_offset &= PCI_MSIX_TABLE_OFFSET; 633 634 msix_tbl = ep->epf_bar[bir]->addr + tbl_offset; 635 msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; 636 msg_data = msix_tbl[(interrupt_num - 1)].msg_data; 637 vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl; 638 639 if (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT) { 640 dev_dbg(pci->dev, "MSI-X entry ctrl set\n"); 641 return -EPERM; 642 } 643 644 msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); 645 ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, 646 map_size); 647 if (ret) 648 return ret; 649 650 writel(msg_data, ep->msi_mem + offset); 651 652 dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); 653 654 return 0; 655 } 656 657 /** 658 * dw_pcie_ep_cleanup - Cleanup DWC EP resources after fundamental reset 659 * @ep: DWC EP device 660 * 661 * Cleans up the DWC EP specific resources like eDMA etc... after fundamental 662 * reset like PERST#. Note that this API is only applicable for drivers 663 * supporting PERST# or any other methods of fundamental reset. 664 */ 665 void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) 666 { 667 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 668 669 dw_pcie_edma_remove(pci); 670 } 671 EXPORT_SYMBOL_GPL(dw_pcie_ep_cleanup); 672 673 /** 674 * dw_pcie_ep_deinit - Deinitialize the endpoint device 675 * @ep: DWC EP device 676 * 677 * Deinitialize the endpoint device. EPC device is not destroyed since that will 678 * be taken care by Devres. 679 */ 680 void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) 681 { 682 struct pci_epc *epc = ep->epc; 683 684 dw_pcie_ep_cleanup(ep); 685 686 pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, 687 epc->mem->window.page_size); 688 689 pci_epc_mem_exit(epc); 690 } 691 EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); 692 693 static unsigned int dw_pcie_ep_find_ext_capability(struct dw_pcie *pci, int cap) 694 { 695 u32 header; 696 int pos = PCI_CFG_SPACE_SIZE; 697 698 while (pos) { 699 header = dw_pcie_readl_dbi(pci, pos); 700 if (PCI_EXT_CAP_ID(header) == cap) 701 return pos; 702 703 pos = PCI_EXT_CAP_NEXT(header); 704 if (!pos) 705 break; 706 } 707 708 return 0; 709 } 710 711 static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) 712 { 713 unsigned int offset; 714 unsigned int nbars; 715 u32 reg, i; 716 717 offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); 718 719 dw_pcie_dbi_ro_wr_en(pci); 720 721 if (offset) { 722 reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); 723 nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >> 724 PCI_REBAR_CTRL_NBAR_SHIFT; 725 726 /* 727 * PCIe r6.0, sec 7.8.6.2 require us to support at least one 728 * size in the range from 1 MB to 512 GB. Advertise support 729 * for 1 MB BAR size only. 730 */ 731 for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) 732 dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, BIT(4)); 733 } 734 735 dw_pcie_setup(pci); 736 dw_pcie_dbi_ro_wr_dis(pci); 737 } 738 739 /** 740 * dw_pcie_ep_init_registers - Initialize DWC EP specific registers 741 * @ep: DWC EP device 742 * 743 * Initialize the registers (CSRs) specific to DWC EP. This API should be called 744 * only when the endpoint receives an active refclk (either from host or 745 * generated locally). 746 */ 747 int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) 748 { 749 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 750 struct dw_pcie_ep_func *ep_func; 751 struct device *dev = pci->dev; 752 struct pci_epc *epc = ep->epc; 753 u32 ptm_cap_base, reg; 754 u8 hdr_type; 755 u8 func_no; 756 void *addr; 757 int ret; 758 759 hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE) & 760 PCI_HEADER_TYPE_MASK; 761 if (hdr_type != PCI_HEADER_TYPE_NORMAL) { 762 dev_err(pci->dev, 763 "PCIe controller is not set to EP mode (hdr_type:0x%x)!\n", 764 hdr_type); 765 return -EIO; 766 } 767 768 dw_pcie_version_detect(pci); 769 770 dw_pcie_iatu_detect(pci); 771 772 ret = dw_pcie_edma_detect(pci); 773 if (ret) 774 return ret; 775 776 if (!ep->ib_window_map) { 777 ep->ib_window_map = devm_bitmap_zalloc(dev, pci->num_ib_windows, 778 GFP_KERNEL); 779 if (!ep->ib_window_map) 780 goto err_remove_edma; 781 } 782 783 if (!ep->ob_window_map) { 784 ep->ob_window_map = devm_bitmap_zalloc(dev, pci->num_ob_windows, 785 GFP_KERNEL); 786 if (!ep->ob_window_map) 787 goto err_remove_edma; 788 } 789 790 if (!ep->outbound_addr) { 791 addr = devm_kcalloc(dev, pci->num_ob_windows, sizeof(phys_addr_t), 792 GFP_KERNEL); 793 if (!addr) 794 goto err_remove_edma; 795 ep->outbound_addr = addr; 796 } 797 798 for (func_no = 0; func_no < epc->max_functions; func_no++) { 799 800 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 801 if (ep_func) 802 continue; 803 804 ep_func = devm_kzalloc(dev, sizeof(*ep_func), GFP_KERNEL); 805 if (!ep_func) 806 goto err_remove_edma; 807 808 ep_func->func_no = func_no; 809 ep_func->msi_cap = dw_pcie_ep_find_capability(ep, func_no, 810 PCI_CAP_ID_MSI); 811 ep_func->msix_cap = dw_pcie_ep_find_capability(ep, func_no, 812 PCI_CAP_ID_MSIX); 813 814 list_add_tail(&ep_func->list, &ep->func_list); 815 } 816 817 if (ep->ops->init) 818 ep->ops->init(ep); 819 820 ptm_cap_base = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_PTM); 821 822 /* 823 * PTM responder capability can be disabled only after disabling 824 * PTM root capability. 825 */ 826 if (ptm_cap_base) { 827 dw_pcie_dbi_ro_wr_en(pci); 828 reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); 829 reg &= ~PCI_PTM_CAP_ROOT; 830 dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); 831 832 reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); 833 reg &= ~(PCI_PTM_CAP_RES | PCI_PTM_GRANULARITY_MASK); 834 dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); 835 dw_pcie_dbi_ro_wr_dis(pci); 836 } 837 838 dw_pcie_ep_init_non_sticky_registers(pci); 839 840 return 0; 841 842 err_remove_edma: 843 dw_pcie_edma_remove(pci); 844 845 return ret; 846 } 847 EXPORT_SYMBOL_GPL(dw_pcie_ep_init_registers); 848 849 /** 850 * dw_pcie_ep_linkup - Notify EPF drivers about Link Up event 851 * @ep: DWC EP device 852 */ 853 void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) 854 { 855 struct pci_epc *epc = ep->epc; 856 857 pci_epc_linkup(epc); 858 } 859 EXPORT_SYMBOL_GPL(dw_pcie_ep_linkup); 860 861 /** 862 * dw_pcie_ep_linkdown - Notify EPF drivers about Link Down event 863 * @ep: DWC EP device 864 * 865 * Non-sticky registers are also initialized before sending the notification to 866 * the EPF drivers. This is needed since the registers need to be initialized 867 * before the link comes back again. 868 */ 869 void dw_pcie_ep_linkdown(struct dw_pcie_ep *ep) 870 { 871 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 872 struct pci_epc *epc = ep->epc; 873 874 /* 875 * Initialize the non-sticky DWC registers as they would've reset post 876 * Link Down. This is specifically needed for drivers not supporting 877 * PERST# as they have no way to reinitialize the registers before the 878 * link comes back again. 879 */ 880 dw_pcie_ep_init_non_sticky_registers(pci); 881 882 pci_epc_linkdown(epc); 883 } 884 EXPORT_SYMBOL_GPL(dw_pcie_ep_linkdown); 885 886 /** 887 * dw_pcie_ep_init - Initialize the endpoint device 888 * @ep: DWC EP device 889 * 890 * Initialize the endpoint device. Allocate resources and create the EPC 891 * device with the endpoint framework. 892 * 893 * Return: 0 if success, errno otherwise. 894 */ 895 int dw_pcie_ep_init(struct dw_pcie_ep *ep) 896 { 897 int ret; 898 struct resource *res; 899 struct pci_epc *epc; 900 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 901 struct device *dev = pci->dev; 902 struct platform_device *pdev = to_platform_device(dev); 903 struct device_node *np = dev->of_node; 904 905 INIT_LIST_HEAD(&ep->func_list); 906 907 ret = dw_pcie_get_resources(pci); 908 if (ret) 909 return ret; 910 911 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); 912 if (!res) 913 return -EINVAL; 914 915 ep->phys_base = res->start; 916 ep->addr_size = resource_size(res); 917 918 if (ep->ops->pre_init) 919 ep->ops->pre_init(ep); 920 921 epc = devm_pci_epc_create(dev, &epc_ops); 922 if (IS_ERR(epc)) { 923 dev_err(dev, "Failed to create epc device\n"); 924 return PTR_ERR(epc); 925 } 926 927 ep->epc = epc; 928 epc_set_drvdata(epc, ep); 929 930 ret = of_property_read_u8(np, "max-functions", &epc->max_functions); 931 if (ret < 0) 932 epc->max_functions = 1; 933 934 ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size, 935 ep->page_size); 936 if (ret < 0) { 937 dev_err(dev, "Failed to initialize address space\n"); 938 return ret; 939 } 940 941 ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys, 942 epc->mem->window.page_size); 943 if (!ep->msi_mem) { 944 ret = -ENOMEM; 945 dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); 946 goto err_exit_epc_mem; 947 } 948 949 return 0; 950 951 err_exit_epc_mem: 952 pci_epc_mem_exit(epc); 953 954 return ret; 955 } 956 EXPORT_SYMBOL_GPL(dw_pcie_ep_init); 957