1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Test driver to test endpoint functionality 4 * 5 * Copyright (C) 2017 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 #include <linux/crc32.h> 10 #include <linux/delay.h> 11 #include <linux/dmaengine.h> 12 #include <linux/io.h> 13 #include <linux/module.h> 14 #include <linux/slab.h> 15 #include <linux/pci_ids.h> 16 #include <linux/random.h> 17 18 #include <linux/pci-epc.h> 19 #include <linux/pci-epf.h> 20 #include <linux/pci_regs.h> 21 22 #define IRQ_TYPE_INTX 0 23 #define IRQ_TYPE_MSI 1 24 #define IRQ_TYPE_MSIX 2 25 26 #define COMMAND_RAISE_INTX_IRQ BIT(0) 27 #define COMMAND_RAISE_MSI_IRQ BIT(1) 28 #define COMMAND_RAISE_MSIX_IRQ BIT(2) 29 #define COMMAND_READ BIT(3) 30 #define COMMAND_WRITE BIT(4) 31 #define COMMAND_COPY BIT(5) 32 33 #define STATUS_READ_SUCCESS BIT(0) 34 #define STATUS_READ_FAIL BIT(1) 35 #define STATUS_WRITE_SUCCESS BIT(2) 36 #define STATUS_WRITE_FAIL BIT(3) 37 #define STATUS_COPY_SUCCESS BIT(4) 38 #define STATUS_COPY_FAIL BIT(5) 39 #define STATUS_IRQ_RAISED BIT(6) 40 #define STATUS_SRC_ADDR_INVALID BIT(7) 41 #define STATUS_DST_ADDR_INVALID BIT(8) 42 43 #define FLAG_USE_DMA BIT(0) 44 45 #define TIMER_RESOLUTION 1 46 47 #define CAP_UNALIGNED_ACCESS BIT(0) 48 #define CAP_MSI BIT(1) 49 #define CAP_MSIX BIT(2) 50 #define CAP_INTX BIT(3) 51 52 static struct workqueue_struct *kpcitest_workqueue; 53 54 struct pci_epf_test { 55 void *reg[PCI_STD_NUM_BARS]; 56 struct pci_epf *epf; 57 enum pci_barno test_reg_bar; 58 size_t msix_table_offset; 59 struct delayed_work cmd_handler; 60 struct dma_chan *dma_chan_tx; 61 struct dma_chan *dma_chan_rx; 62 struct dma_chan *transfer_chan; 63 dma_cookie_t transfer_cookie; 64 enum dma_status transfer_status; 65 struct completion transfer_complete; 66 bool dma_supported; 67 bool dma_private; 68 const struct pci_epc_features *epc_features; 69 }; 70 71 struct pci_epf_test_reg { 72 __le32 magic; 73 __le32 command; 74 __le32 status; 75 __le64 src_addr; 76 __le64 dst_addr; 77 __le32 size; 78 __le32 checksum; 79 __le32 irq_type; 80 __le32 irq_number; 81 __le32 flags; 82 __le32 caps; 83 } __packed; 84 85 static struct pci_epf_header test_header = { 86 .vendorid = PCI_ANY_ID, 87 .deviceid = PCI_ANY_ID, 88 .baseclass_code = PCI_CLASS_OTHERS, 89 .interrupt_pin = PCI_INTERRUPT_INTA, 90 }; 91 92 static size_t bar_size[] = { 512, 512, 1024, 16384, 131072, 1048576 }; 93 94 static void pci_epf_test_dma_callback(void *param) 95 { 96 struct pci_epf_test *epf_test = param; 97 struct dma_tx_state state; 98 99 epf_test->transfer_status = 100 dmaengine_tx_status(epf_test->transfer_chan, 101 epf_test->transfer_cookie, &state); 102 if (epf_test->transfer_status == DMA_COMPLETE || 103 epf_test->transfer_status == DMA_ERROR) 104 complete(&epf_test->transfer_complete); 105 } 106 107 /** 108 * pci_epf_test_data_transfer() - Function that uses dmaengine API to transfer 109 * data between PCIe EP and remote PCIe RC 110 * @epf_test: the EPF test device that performs the data transfer operation 111 * @dma_dst: The destination address of the data transfer. It can be a physical 112 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs. 113 * @dma_src: The source address of the data transfer. It can be a physical 114 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs. 115 * @len: The size of the data transfer 116 * @dma_remote: remote RC physical address 117 * @dir: DMA transfer direction 118 * 119 * Function that uses dmaengine API to transfer data between PCIe EP and remote 120 * PCIe RC. The source and destination address can be a physical address given 121 * by pci_epc_mem_alloc_addr or the one obtained using DMA mapping APIs. 122 * 123 * The function returns '0' on success and negative value on failure. 124 */ 125 static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test, 126 dma_addr_t dma_dst, dma_addr_t dma_src, 127 size_t len, dma_addr_t dma_remote, 128 enum dma_transfer_direction dir) 129 { 130 struct dma_chan *chan = (dir == DMA_MEM_TO_DEV) ? 131 epf_test->dma_chan_tx : epf_test->dma_chan_rx; 132 dma_addr_t dma_local = (dir == DMA_MEM_TO_DEV) ? dma_src : dma_dst; 133 enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; 134 struct pci_epf *epf = epf_test->epf; 135 struct dma_async_tx_descriptor *tx; 136 struct dma_slave_config sconf = {}; 137 struct device *dev = &epf->dev; 138 int ret; 139 140 if (IS_ERR_OR_NULL(chan)) { 141 dev_err(dev, "Invalid DMA memcpy channel\n"); 142 return -EINVAL; 143 } 144 145 if (epf_test->dma_private) { 146 sconf.direction = dir; 147 if (dir == DMA_MEM_TO_DEV) 148 sconf.dst_addr = dma_remote; 149 else 150 sconf.src_addr = dma_remote; 151 152 if (dmaengine_slave_config(chan, &sconf)) { 153 dev_err(dev, "DMA slave config fail\n"); 154 return -EIO; 155 } 156 tx = dmaengine_prep_slave_single(chan, dma_local, len, dir, 157 flags); 158 } else { 159 tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len, 160 flags); 161 } 162 163 if (!tx) { 164 dev_err(dev, "Failed to prepare DMA memcpy\n"); 165 return -EIO; 166 } 167 168 reinit_completion(&epf_test->transfer_complete); 169 epf_test->transfer_chan = chan; 170 tx->callback = pci_epf_test_dma_callback; 171 tx->callback_param = epf_test; 172 epf_test->transfer_cookie = dmaengine_submit(tx); 173 174 ret = dma_submit_error(epf_test->transfer_cookie); 175 if (ret) { 176 dev_err(dev, "Failed to do DMA tx_submit %d\n", ret); 177 goto terminate; 178 } 179 180 dma_async_issue_pending(chan); 181 ret = wait_for_completion_interruptible(&epf_test->transfer_complete); 182 if (ret < 0) { 183 dev_err(dev, "DMA wait_for_completion interrupted\n"); 184 goto terminate; 185 } 186 187 if (epf_test->transfer_status == DMA_ERROR) { 188 dev_err(dev, "DMA transfer failed\n"); 189 ret = -EIO; 190 } 191 192 terminate: 193 dmaengine_terminate_sync(chan); 194 195 return ret; 196 } 197 198 struct epf_dma_filter { 199 struct device *dev; 200 u32 dma_mask; 201 }; 202 203 static bool epf_dma_filter_fn(struct dma_chan *chan, void *node) 204 { 205 struct epf_dma_filter *filter = node; 206 struct dma_slave_caps caps; 207 208 memset(&caps, 0, sizeof(caps)); 209 dma_get_slave_caps(chan, &caps); 210 211 return chan->device->dev == filter->dev 212 && (filter->dma_mask & caps.directions); 213 } 214 215 /** 216 * pci_epf_test_init_dma_chan() - Function to initialize EPF test DMA channel 217 * @epf_test: the EPF test device that performs data transfer operation 218 * 219 * Function to initialize EPF test DMA channel. 220 */ 221 static int pci_epf_test_init_dma_chan(struct pci_epf_test *epf_test) 222 { 223 struct pci_epf *epf = epf_test->epf; 224 struct device *dev = &epf->dev; 225 struct epf_dma_filter filter; 226 struct dma_chan *dma_chan; 227 dma_cap_mask_t mask; 228 int ret; 229 230 filter.dev = epf->epc->dev.parent; 231 filter.dma_mask = BIT(DMA_DEV_TO_MEM); 232 233 dma_cap_zero(mask); 234 dma_cap_set(DMA_SLAVE, mask); 235 dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter); 236 if (!dma_chan) { 237 dev_info(dev, "Failed to get private DMA rx channel. Falling back to generic one\n"); 238 goto fail_back_tx; 239 } 240 241 epf_test->dma_chan_rx = dma_chan; 242 243 filter.dma_mask = BIT(DMA_MEM_TO_DEV); 244 dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter); 245 246 if (!dma_chan) { 247 dev_info(dev, "Failed to get private DMA tx channel. Falling back to generic one\n"); 248 goto fail_back_rx; 249 } 250 251 epf_test->dma_chan_tx = dma_chan; 252 epf_test->dma_private = true; 253 254 init_completion(&epf_test->transfer_complete); 255 256 return 0; 257 258 fail_back_rx: 259 dma_release_channel(epf_test->dma_chan_rx); 260 epf_test->dma_chan_rx = NULL; 261 262 fail_back_tx: 263 dma_cap_zero(mask); 264 dma_cap_set(DMA_MEMCPY, mask); 265 266 dma_chan = dma_request_chan_by_mask(&mask); 267 if (IS_ERR(dma_chan)) { 268 ret = PTR_ERR(dma_chan); 269 if (ret != -EPROBE_DEFER) 270 dev_err(dev, "Failed to get DMA channel\n"); 271 return ret; 272 } 273 init_completion(&epf_test->transfer_complete); 274 275 epf_test->dma_chan_tx = epf_test->dma_chan_rx = dma_chan; 276 277 return 0; 278 } 279 280 /** 281 * pci_epf_test_clean_dma_chan() - Function to cleanup EPF test DMA channel 282 * @epf_test: the EPF test device that performs data transfer operation 283 * 284 * Helper to cleanup EPF test DMA channel. 285 */ 286 static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test) 287 { 288 if (!epf_test->dma_supported) 289 return; 290 291 dma_release_channel(epf_test->dma_chan_tx); 292 if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) { 293 epf_test->dma_chan_tx = NULL; 294 epf_test->dma_chan_rx = NULL; 295 return; 296 } 297 298 dma_release_channel(epf_test->dma_chan_rx); 299 epf_test->dma_chan_rx = NULL; 300 } 301 302 static void pci_epf_test_print_rate(struct pci_epf_test *epf_test, 303 const char *op, u64 size, 304 struct timespec64 *start, 305 struct timespec64 *end, bool dma) 306 { 307 struct timespec64 ts = timespec64_sub(*end, *start); 308 u64 rate = 0, ns; 309 310 /* calculate the rate */ 311 ns = timespec64_to_ns(&ts); 312 if (ns) 313 rate = div64_u64(size * NSEC_PER_SEC, ns * 1000); 314 315 dev_info(&epf_test->epf->dev, 316 "%s => Size: %llu B, DMA: %s, Time: %llu.%09u s, Rate: %llu KB/s\n", 317 op, size, dma ? "YES" : "NO", 318 (u64)ts.tv_sec, (u32)ts.tv_nsec, rate); 319 } 320 321 static void pci_epf_test_copy(struct pci_epf_test *epf_test, 322 struct pci_epf_test_reg *reg) 323 { 324 int ret = 0; 325 struct timespec64 start, end; 326 struct pci_epf *epf = epf_test->epf; 327 struct pci_epc *epc = epf->epc; 328 struct device *dev = &epf->dev; 329 struct pci_epc_map src_map, dst_map; 330 u64 src_addr = le64_to_cpu(reg->src_addr); 331 u64 dst_addr = le64_to_cpu(reg->dst_addr); 332 size_t orig_size, copy_size; 333 ssize_t map_size = 0; 334 u32 flags = le32_to_cpu(reg->flags); 335 u32 status = 0; 336 void *copy_buf = NULL, *buf; 337 338 orig_size = copy_size = le32_to_cpu(reg->size); 339 340 if (flags & FLAG_USE_DMA) { 341 if (!dma_has_cap(DMA_MEMCPY, epf_test->dma_chan_tx->device->cap_mask)) { 342 dev_err(dev, "DMA controller doesn't support MEMCPY\n"); 343 ret = -EINVAL; 344 goto set_status; 345 } 346 } else { 347 copy_buf = kzalloc(copy_size, GFP_KERNEL); 348 if (!copy_buf) { 349 ret = -ENOMEM; 350 goto set_status; 351 } 352 buf = copy_buf; 353 } 354 355 while (copy_size) { 356 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, 357 src_addr, copy_size, &src_map); 358 if (ret) { 359 dev_err(dev, "Failed to map source address\n"); 360 status = STATUS_SRC_ADDR_INVALID; 361 goto free_buf; 362 } 363 364 ret = pci_epc_mem_map(epf->epc, epf->func_no, epf->vfunc_no, 365 dst_addr, copy_size, &dst_map); 366 if (ret) { 367 dev_err(dev, "Failed to map destination address\n"); 368 status = STATUS_DST_ADDR_INVALID; 369 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, 370 &src_map); 371 goto free_buf; 372 } 373 374 map_size = min_t(size_t, dst_map.pci_size, src_map.pci_size); 375 376 ktime_get_ts64(&start); 377 if (flags & FLAG_USE_DMA) { 378 ret = pci_epf_test_data_transfer(epf_test, 379 dst_map.phys_addr, src_map.phys_addr, 380 map_size, 0, DMA_MEM_TO_MEM); 381 if (ret) { 382 dev_err(dev, "Data transfer failed\n"); 383 goto unmap; 384 } 385 } else { 386 memcpy_fromio(buf, src_map.virt_addr, map_size); 387 memcpy_toio(dst_map.virt_addr, buf, map_size); 388 buf += map_size; 389 } 390 ktime_get_ts64(&end); 391 392 copy_size -= map_size; 393 src_addr += map_size; 394 dst_addr += map_size; 395 396 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); 397 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); 398 map_size = 0; 399 } 400 401 pci_epf_test_print_rate(epf_test, "COPY", orig_size, &start, &end, 402 flags & FLAG_USE_DMA); 403 404 unmap: 405 if (map_size) { 406 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); 407 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); 408 } 409 410 free_buf: 411 kfree(copy_buf); 412 413 set_status: 414 if (!ret) 415 status |= STATUS_COPY_SUCCESS; 416 else 417 status |= STATUS_COPY_FAIL; 418 reg->status = cpu_to_le32(status); 419 } 420 421 static void pci_epf_test_read(struct pci_epf_test *epf_test, 422 struct pci_epf_test_reg *reg) 423 { 424 int ret = 0; 425 void *src_buf, *buf; 426 u32 crc32; 427 struct pci_epc_map map; 428 phys_addr_t dst_phys_addr; 429 struct timespec64 start, end; 430 struct pci_epf *epf = epf_test->epf; 431 struct pci_epc *epc = epf->epc; 432 struct device *dev = &epf->dev; 433 struct device *dma_dev = epf->epc->dev.parent; 434 u64 src_addr = le64_to_cpu(reg->src_addr); 435 size_t orig_size, src_size; 436 ssize_t map_size = 0; 437 u32 flags = le32_to_cpu(reg->flags); 438 u32 checksum = le32_to_cpu(reg->checksum); 439 u32 status = 0; 440 441 orig_size = src_size = le32_to_cpu(reg->size); 442 443 src_buf = kzalloc(src_size, GFP_KERNEL); 444 if (!src_buf) { 445 ret = -ENOMEM; 446 goto set_status; 447 } 448 buf = src_buf; 449 450 while (src_size) { 451 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, 452 src_addr, src_size, &map); 453 if (ret) { 454 dev_err(dev, "Failed to map address\n"); 455 status = STATUS_SRC_ADDR_INVALID; 456 goto free_buf; 457 } 458 459 map_size = map.pci_size; 460 if (flags & FLAG_USE_DMA) { 461 dst_phys_addr = dma_map_single(dma_dev, buf, map_size, 462 DMA_FROM_DEVICE); 463 if (dma_mapping_error(dma_dev, dst_phys_addr)) { 464 dev_err(dev, 465 "Failed to map destination buffer addr\n"); 466 ret = -ENOMEM; 467 goto unmap; 468 } 469 470 ktime_get_ts64(&start); 471 ret = pci_epf_test_data_transfer(epf_test, 472 dst_phys_addr, map.phys_addr, 473 map_size, src_addr, DMA_DEV_TO_MEM); 474 if (ret) 475 dev_err(dev, "Data transfer failed\n"); 476 ktime_get_ts64(&end); 477 478 dma_unmap_single(dma_dev, dst_phys_addr, map_size, 479 DMA_FROM_DEVICE); 480 481 if (ret) 482 goto unmap; 483 } else { 484 ktime_get_ts64(&start); 485 memcpy_fromio(buf, map.virt_addr, map_size); 486 ktime_get_ts64(&end); 487 } 488 489 src_size -= map_size; 490 src_addr += map_size; 491 buf += map_size; 492 493 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); 494 map_size = 0; 495 } 496 497 pci_epf_test_print_rate(epf_test, "READ", orig_size, &start, &end, 498 flags & FLAG_USE_DMA); 499 500 crc32 = crc32_le(~0, src_buf, orig_size); 501 if (crc32 != checksum) 502 ret = -EIO; 503 504 unmap: 505 if (map_size) 506 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); 507 508 free_buf: 509 kfree(src_buf); 510 511 set_status: 512 if (!ret) 513 status |= STATUS_READ_SUCCESS; 514 else 515 status |= STATUS_READ_FAIL; 516 reg->status = cpu_to_le32(status); 517 } 518 519 static void pci_epf_test_write(struct pci_epf_test *epf_test, 520 struct pci_epf_test_reg *reg) 521 { 522 int ret = 0; 523 void *dst_buf, *buf; 524 struct pci_epc_map map; 525 phys_addr_t src_phys_addr; 526 struct timespec64 start, end; 527 struct pci_epf *epf = epf_test->epf; 528 struct pci_epc *epc = epf->epc; 529 struct device *dev = &epf->dev; 530 struct device *dma_dev = epf->epc->dev.parent; 531 u64 dst_addr = le64_to_cpu(reg->dst_addr); 532 size_t orig_size, dst_size; 533 ssize_t map_size = 0; 534 u32 flags = le32_to_cpu(reg->flags); 535 u32 status = 0; 536 537 orig_size = dst_size = le32_to_cpu(reg->size); 538 539 dst_buf = kzalloc(dst_size, GFP_KERNEL); 540 if (!dst_buf) { 541 ret = -ENOMEM; 542 goto set_status; 543 } 544 get_random_bytes(dst_buf, dst_size); 545 reg->checksum = cpu_to_le32(crc32_le(~0, dst_buf, dst_size)); 546 buf = dst_buf; 547 548 while (dst_size) { 549 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, 550 dst_addr, dst_size, &map); 551 if (ret) { 552 dev_err(dev, "Failed to map address\n"); 553 status = STATUS_DST_ADDR_INVALID; 554 goto free_buf; 555 } 556 557 map_size = map.pci_size; 558 if (flags & FLAG_USE_DMA) { 559 src_phys_addr = dma_map_single(dma_dev, buf, map_size, 560 DMA_TO_DEVICE); 561 if (dma_mapping_error(dma_dev, src_phys_addr)) { 562 dev_err(dev, 563 "Failed to map source buffer addr\n"); 564 ret = -ENOMEM; 565 goto unmap; 566 } 567 568 ktime_get_ts64(&start); 569 570 ret = pci_epf_test_data_transfer(epf_test, 571 map.phys_addr, src_phys_addr, 572 map_size, dst_addr, 573 DMA_MEM_TO_DEV); 574 if (ret) 575 dev_err(dev, "Data transfer failed\n"); 576 ktime_get_ts64(&end); 577 578 dma_unmap_single(dma_dev, src_phys_addr, map_size, 579 DMA_TO_DEVICE); 580 581 if (ret) 582 goto unmap; 583 } else { 584 ktime_get_ts64(&start); 585 memcpy_toio(map.virt_addr, buf, map_size); 586 ktime_get_ts64(&end); 587 } 588 589 dst_size -= map_size; 590 dst_addr += map_size; 591 buf += map_size; 592 593 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); 594 map_size = 0; 595 } 596 597 pci_epf_test_print_rate(epf_test, "WRITE", orig_size, &start, &end, 598 flags & FLAG_USE_DMA); 599 600 /* 601 * wait 1ms inorder for the write to complete. Without this delay L3 602 * error in observed in the host system. 603 */ 604 usleep_range(1000, 2000); 605 606 unmap: 607 if (map_size) 608 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); 609 610 free_buf: 611 kfree(dst_buf); 612 613 set_status: 614 if (!ret) 615 status |= STATUS_WRITE_SUCCESS; 616 else 617 status |= STATUS_WRITE_FAIL; 618 reg->status = cpu_to_le32(status); 619 } 620 621 static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test, 622 struct pci_epf_test_reg *reg) 623 { 624 struct pci_epf *epf = epf_test->epf; 625 struct device *dev = &epf->dev; 626 struct pci_epc *epc = epf->epc; 627 u32 status = le32_to_cpu(reg->status); 628 u32 irq_number = le32_to_cpu(reg->irq_number); 629 u32 irq_type = le32_to_cpu(reg->irq_type); 630 int count; 631 632 /* 633 * Set the status before raising the IRQ to ensure that the host sees 634 * the updated value when it gets the IRQ. 635 */ 636 status |= STATUS_IRQ_RAISED; 637 WRITE_ONCE(reg->status, cpu_to_le32(status)); 638 639 switch (irq_type) { 640 case IRQ_TYPE_INTX: 641 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 642 PCI_IRQ_INTX, 0); 643 break; 644 case IRQ_TYPE_MSI: 645 count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no); 646 if (irq_number > count || count <= 0) { 647 dev_err(dev, "Invalid MSI IRQ number %d / %d\n", 648 irq_number, count); 649 return; 650 } 651 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 652 PCI_IRQ_MSI, irq_number); 653 break; 654 case IRQ_TYPE_MSIX: 655 count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no); 656 if (irq_number > count || count <= 0) { 657 dev_err(dev, "Invalid MSI-X IRQ number %d / %d\n", 658 irq_number, count); 659 return; 660 } 661 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 662 PCI_IRQ_MSIX, irq_number); 663 break; 664 default: 665 dev_err(dev, "Failed to raise IRQ, unknown type\n"); 666 break; 667 } 668 } 669 670 static void pci_epf_test_cmd_handler(struct work_struct *work) 671 { 672 u32 command; 673 struct pci_epf_test *epf_test = container_of(work, struct pci_epf_test, 674 cmd_handler.work); 675 struct pci_epf *epf = epf_test->epf; 676 struct device *dev = &epf->dev; 677 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 678 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 679 u32 irq_type = le32_to_cpu(reg->irq_type); 680 681 command = le32_to_cpu(READ_ONCE(reg->command)); 682 if (!command) 683 goto reset_handler; 684 685 WRITE_ONCE(reg->command, 0); 686 WRITE_ONCE(reg->status, 0); 687 688 if ((le32_to_cpu(READ_ONCE(reg->flags)) & FLAG_USE_DMA) && 689 !epf_test->dma_supported) { 690 dev_err(dev, "Cannot transfer data using DMA\n"); 691 goto reset_handler; 692 } 693 694 if (irq_type > IRQ_TYPE_MSIX) { 695 dev_err(dev, "Failed to detect IRQ type\n"); 696 goto reset_handler; 697 } 698 699 switch (command) { 700 case COMMAND_RAISE_INTX_IRQ: 701 case COMMAND_RAISE_MSI_IRQ: 702 case COMMAND_RAISE_MSIX_IRQ: 703 pci_epf_test_raise_irq(epf_test, reg); 704 break; 705 case COMMAND_WRITE: 706 pci_epf_test_write(epf_test, reg); 707 pci_epf_test_raise_irq(epf_test, reg); 708 break; 709 case COMMAND_READ: 710 pci_epf_test_read(epf_test, reg); 711 pci_epf_test_raise_irq(epf_test, reg); 712 break; 713 case COMMAND_COPY: 714 pci_epf_test_copy(epf_test, reg); 715 pci_epf_test_raise_irq(epf_test, reg); 716 break; 717 default: 718 dev_err(dev, "Invalid command 0x%x\n", command); 719 break; 720 } 721 722 reset_handler: 723 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, 724 msecs_to_jiffies(1)); 725 } 726 727 static int pci_epf_test_set_bar(struct pci_epf *epf) 728 { 729 int bar, ret; 730 struct pci_epc *epc = epf->epc; 731 struct device *dev = &epf->dev; 732 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 733 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 734 735 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 736 if (!epf_test->reg[bar]) 737 continue; 738 739 ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, 740 &epf->bar[bar]); 741 if (ret) { 742 pci_epf_free_space(epf, epf_test->reg[bar], bar, 743 PRIMARY_INTERFACE); 744 epf_test->reg[bar] = NULL; 745 dev_err(dev, "Failed to set BAR%d\n", bar); 746 if (bar == test_reg_bar) 747 return ret; 748 } 749 } 750 751 return 0; 752 } 753 754 static void pci_epf_test_clear_bar(struct pci_epf *epf) 755 { 756 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 757 struct pci_epc *epc = epf->epc; 758 int bar; 759 760 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 761 if (!epf_test->reg[bar]) 762 continue; 763 764 pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, 765 &epf->bar[bar]); 766 } 767 } 768 769 static void pci_epf_test_set_capabilities(struct pci_epf *epf) 770 { 771 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 772 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 773 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 774 struct pci_epc *epc = epf->epc; 775 u32 caps = 0; 776 777 if (epc->ops->align_addr) 778 caps |= CAP_UNALIGNED_ACCESS; 779 780 if (epf_test->epc_features->msi_capable) 781 caps |= CAP_MSI; 782 783 if (epf_test->epc_features->msix_capable) 784 caps |= CAP_MSIX; 785 786 if (epf_test->epc_features->intx_capable) 787 caps |= CAP_INTX; 788 789 reg->caps = cpu_to_le32(caps); 790 } 791 792 static int pci_epf_test_epc_init(struct pci_epf *epf) 793 { 794 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 795 struct pci_epf_header *header = epf->header; 796 const struct pci_epc_features *epc_features = epf_test->epc_features; 797 struct pci_epc *epc = epf->epc; 798 struct device *dev = &epf->dev; 799 bool linkup_notifier = false; 800 int ret; 801 802 epf_test->dma_supported = true; 803 804 ret = pci_epf_test_init_dma_chan(epf_test); 805 if (ret) 806 epf_test->dma_supported = false; 807 808 if (epf->vfunc_no <= 1) { 809 ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); 810 if (ret) { 811 dev_err(dev, "Configuration header write failed\n"); 812 return ret; 813 } 814 } 815 816 pci_epf_test_set_capabilities(epf); 817 818 ret = pci_epf_test_set_bar(epf); 819 if (ret) 820 return ret; 821 822 if (epc_features->msi_capable) { 823 ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no, 824 epf->msi_interrupts); 825 if (ret) { 826 dev_err(dev, "MSI configuration failed\n"); 827 return ret; 828 } 829 } 830 831 if (epc_features->msix_capable) { 832 ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no, 833 epf->msix_interrupts, 834 epf_test->test_reg_bar, 835 epf_test->msix_table_offset); 836 if (ret) { 837 dev_err(dev, "MSI-X configuration failed\n"); 838 return ret; 839 } 840 } 841 842 linkup_notifier = epc_features->linkup_notifier; 843 if (!linkup_notifier) 844 queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work); 845 846 return 0; 847 } 848 849 static void pci_epf_test_epc_deinit(struct pci_epf *epf) 850 { 851 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 852 853 cancel_delayed_work_sync(&epf_test->cmd_handler); 854 pci_epf_test_clean_dma_chan(epf_test); 855 pci_epf_test_clear_bar(epf); 856 } 857 858 static int pci_epf_test_link_up(struct pci_epf *epf) 859 { 860 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 861 862 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, 863 msecs_to_jiffies(1)); 864 865 return 0; 866 } 867 868 static int pci_epf_test_link_down(struct pci_epf *epf) 869 { 870 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 871 872 cancel_delayed_work_sync(&epf_test->cmd_handler); 873 874 return 0; 875 } 876 877 static const struct pci_epc_event_ops pci_epf_test_event_ops = { 878 .epc_init = pci_epf_test_epc_init, 879 .epc_deinit = pci_epf_test_epc_deinit, 880 .link_up = pci_epf_test_link_up, 881 .link_down = pci_epf_test_link_down, 882 }; 883 884 static int pci_epf_test_alloc_space(struct pci_epf *epf) 885 { 886 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 887 struct device *dev = &epf->dev; 888 size_t msix_table_size = 0; 889 size_t test_reg_bar_size; 890 size_t pba_size = 0; 891 void *base; 892 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 893 enum pci_barno bar; 894 const struct pci_epc_features *epc_features = epf_test->epc_features; 895 size_t test_reg_size; 896 897 test_reg_bar_size = ALIGN(sizeof(struct pci_epf_test_reg), 128); 898 899 if (epc_features->msix_capable) { 900 msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts; 901 epf_test->msix_table_offset = test_reg_bar_size; 902 /* Align to QWORD or 8 Bytes */ 903 pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8); 904 } 905 test_reg_size = test_reg_bar_size + msix_table_size + pba_size; 906 907 base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar, 908 epc_features, PRIMARY_INTERFACE); 909 if (!base) { 910 dev_err(dev, "Failed to allocated register space\n"); 911 return -ENOMEM; 912 } 913 epf_test->reg[test_reg_bar] = base; 914 915 for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar++) { 916 bar = pci_epc_get_next_free_bar(epc_features, bar); 917 if (bar == NO_BAR) 918 break; 919 920 if (bar == test_reg_bar) 921 continue; 922 923 base = pci_epf_alloc_space(epf, bar_size[bar], bar, 924 epc_features, PRIMARY_INTERFACE); 925 if (!base) 926 dev_err(dev, "Failed to allocate space for BAR%d\n", 927 bar); 928 epf_test->reg[bar] = base; 929 } 930 931 return 0; 932 } 933 934 static void pci_epf_test_free_space(struct pci_epf *epf) 935 { 936 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 937 int bar; 938 939 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 940 if (!epf_test->reg[bar]) 941 continue; 942 943 pci_epf_free_space(epf, epf_test->reg[bar], bar, 944 PRIMARY_INTERFACE); 945 epf_test->reg[bar] = NULL; 946 } 947 } 948 949 static int pci_epf_test_bind(struct pci_epf *epf) 950 { 951 int ret; 952 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 953 const struct pci_epc_features *epc_features; 954 enum pci_barno test_reg_bar = BAR_0; 955 struct pci_epc *epc = epf->epc; 956 957 if (WARN_ON_ONCE(!epc)) 958 return -EINVAL; 959 960 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no); 961 if (!epc_features) { 962 dev_err(&epf->dev, "epc_features not implemented\n"); 963 return -EOPNOTSUPP; 964 } 965 966 test_reg_bar = pci_epc_get_first_free_bar(epc_features); 967 if (test_reg_bar < 0) 968 return -EINVAL; 969 970 epf_test->test_reg_bar = test_reg_bar; 971 epf_test->epc_features = epc_features; 972 973 ret = pci_epf_test_alloc_space(epf); 974 if (ret) 975 return ret; 976 977 return 0; 978 } 979 980 static void pci_epf_test_unbind(struct pci_epf *epf) 981 { 982 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 983 struct pci_epc *epc = epf->epc; 984 985 cancel_delayed_work_sync(&epf_test->cmd_handler); 986 if (epc->init_complete) { 987 pci_epf_test_clean_dma_chan(epf_test); 988 pci_epf_test_clear_bar(epf); 989 } 990 pci_epf_test_free_space(epf); 991 } 992 993 static const struct pci_epf_device_id pci_epf_test_ids[] = { 994 { 995 .name = "pci_epf_test", 996 }, 997 {}, 998 }; 999 1000 static int pci_epf_test_probe(struct pci_epf *epf, 1001 const struct pci_epf_device_id *id) 1002 { 1003 struct pci_epf_test *epf_test; 1004 struct device *dev = &epf->dev; 1005 1006 epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL); 1007 if (!epf_test) 1008 return -ENOMEM; 1009 1010 epf->header = &test_header; 1011 epf_test->epf = epf; 1012 1013 INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler); 1014 1015 epf->event_ops = &pci_epf_test_event_ops; 1016 1017 epf_set_drvdata(epf, epf_test); 1018 return 0; 1019 } 1020 1021 static const struct pci_epf_ops ops = { 1022 .unbind = pci_epf_test_unbind, 1023 .bind = pci_epf_test_bind, 1024 }; 1025 1026 static struct pci_epf_driver test_driver = { 1027 .driver.name = "pci_epf_test", 1028 .probe = pci_epf_test_probe, 1029 .id_table = pci_epf_test_ids, 1030 .ops = &ops, 1031 .owner = THIS_MODULE, 1032 }; 1033 1034 static int __init pci_epf_test_init(void) 1035 { 1036 int ret; 1037 1038 kpcitest_workqueue = alloc_workqueue("kpcitest", 1039 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1040 if (!kpcitest_workqueue) { 1041 pr_err("Failed to allocate the kpcitest work queue\n"); 1042 return -ENOMEM; 1043 } 1044 1045 ret = pci_epf_register_driver(&test_driver); 1046 if (ret) { 1047 destroy_workqueue(kpcitest_workqueue); 1048 pr_err("Failed to register pci epf test driver --> %d\n", ret); 1049 return ret; 1050 } 1051 1052 return 0; 1053 } 1054 module_init(pci_epf_test_init); 1055 1056 static void __exit pci_epf_test_exit(void) 1057 { 1058 if (kpcitest_workqueue) 1059 destroy_workqueue(kpcitest_workqueue); 1060 pci_epf_unregister_driver(&test_driver); 1061 } 1062 module_exit(pci_epf_test_exit); 1063 1064 MODULE_DESCRIPTION("PCI EPF TEST DRIVER"); 1065 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 1066 MODULE_LICENSE("GPL v2"); 1067