1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Host side endpoint driver to implement Non-Transparent Bridge functionality 4 * 5 * Copyright (C) 2020 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/delay.h> 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/slab.h> 14 #include <linux/ntb.h> 15 16 #define NTB_EPF_COMMAND 0x0 17 #define CMD_CONFIGURE_DOORBELL 1 18 #define CMD_TEARDOWN_DOORBELL 2 19 #define CMD_CONFIGURE_MW 3 20 #define CMD_TEARDOWN_MW 4 21 #define CMD_LINK_UP 5 22 #define CMD_LINK_DOWN 6 23 24 #define NTB_EPF_ARGUMENT 0x4 25 #define MSIX_ENABLE BIT(16) 26 27 #define NTB_EPF_CMD_STATUS 0x8 28 #define COMMAND_STATUS_OK 1 29 #define COMMAND_STATUS_ERROR 2 30 31 #define NTB_EPF_LINK_STATUS 0x0A 32 #define LINK_STATUS_UP BIT(0) 33 34 #define NTB_EPF_TOPOLOGY 0x0C 35 #define NTB_EPF_LOWER_ADDR 0x10 36 #define NTB_EPF_UPPER_ADDR 0x14 37 #define NTB_EPF_LOWER_SIZE 0x18 38 #define NTB_EPF_UPPER_SIZE 0x1C 39 #define NTB_EPF_MW_COUNT 0x20 40 #define NTB_EPF_MW1_OFFSET 0x24 41 #define NTB_EPF_SPAD_OFFSET 0x28 42 #define NTB_EPF_SPAD_COUNT 0x2C 43 #define NTB_EPF_DB_ENTRY_SIZE 0x30 44 #define NTB_EPF_DB_DATA(n) (0x34 + (n) * 4) 45 #define NTB_EPF_DB_OFFSET(n) (0xB4 + (n) * 4) 46 47 /* 48 * Legacy doorbell slot layout when paired with pci-epf-*ntb: 49 * 50 * slot 0 : reserved for link events 51 * slot 1 : unused (historical extra offset) 52 * slot 2 : DB#0 53 * slot 3 : DB#1 54 * ... 55 * 56 * Thus, NTB_EPF_MIN_DB_COUNT=3 means that we at least create vectors for 57 * doorbells DB#0 and DB#1. 58 */ 59 #define NTB_EPF_MIN_DB_COUNT 3 60 #define NTB_EPF_MAX_DB_COUNT 31 61 62 #define NTB_EPF_COMMAND_TIMEOUT 1000 /* 1 Sec */ 63 64 enum pci_barno { 65 NO_BAR = -1, 66 BAR_0, 67 BAR_1, 68 BAR_2, 69 BAR_3, 70 BAR_4, 71 BAR_5, 72 }; 73 74 enum epf_ntb_bar { 75 BAR_CONFIG, 76 BAR_PEER_SPAD, 77 BAR_DB, 78 BAR_MW1, 79 BAR_MW2, 80 BAR_MW3, 81 BAR_MW4, 82 NTB_BAR_NUM, 83 }; 84 85 enum epf_irq_slot { 86 EPF_IRQ_LINK = 0, 87 EPF_IRQ_RESERVED_DB, /* Historically skipped slot */ 88 EPF_IRQ_DB_START, 89 }; 90 91 #define NTB_EPF_MAX_MW_COUNT (NTB_BAR_NUM - BAR_MW1) 92 93 struct ntb_epf_dev; 94 95 struct ntb_epf_irq_ctx { 96 struct ntb_epf_dev *ndev; 97 unsigned int irq_no; 98 }; 99 100 struct ntb_epf_dev { 101 struct ntb_dev ntb; 102 struct device *dev; 103 /* Mutex to protect providing commands to NTB EPF */ 104 struct mutex cmd_lock; 105 106 const enum pci_barno *barno_map; 107 108 unsigned int mw_count; 109 unsigned int spad_count; 110 unsigned int db_count; 111 112 void __iomem *ctrl_reg; 113 void __iomem *db_reg; 114 void __iomem *peer_spad_reg; 115 116 unsigned int self_spad; 117 unsigned int peer_spad; 118 119 atomic64_t db_val; 120 u64 db_valid_mask; 121 struct ntb_epf_irq_ctx irq_ctx[NTB_EPF_MAX_DB_COUNT + 1]; 122 }; 123 124 #define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb) 125 126 static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command, 127 u32 argument) 128 { 129 ktime_t timeout; 130 bool timedout; 131 int ret = 0; 132 u32 status; 133 134 mutex_lock(&ndev->cmd_lock); 135 writel(argument, ndev->ctrl_reg + NTB_EPF_ARGUMENT); 136 writel(command, ndev->ctrl_reg + NTB_EPF_COMMAND); 137 138 timeout = ktime_add_ms(ktime_get(), NTB_EPF_COMMAND_TIMEOUT); 139 while (1) { 140 timedout = ktime_after(ktime_get(), timeout); 141 status = readw(ndev->ctrl_reg + NTB_EPF_CMD_STATUS); 142 143 if (status == COMMAND_STATUS_ERROR) { 144 ret = -EINVAL; 145 break; 146 } 147 148 if (status == COMMAND_STATUS_OK) 149 break; 150 151 if (WARN_ON(timedout)) { 152 ret = -ETIMEDOUT; 153 break; 154 } 155 156 usleep_range(5, 10); 157 } 158 159 writew(0, ndev->ctrl_reg + NTB_EPF_CMD_STATUS); 160 mutex_unlock(&ndev->cmd_lock); 161 162 return ret; 163 } 164 165 static int ntb_epf_mw_to_bar(struct ntb_epf_dev *ndev, int idx) 166 { 167 struct device *dev = ndev->dev; 168 169 if (idx < 0 || idx > ndev->mw_count) { 170 dev_err(dev, "Unsupported Memory Window index %d\n", idx); 171 return -EINVAL; 172 } 173 174 return ndev->barno_map[BAR_MW1 + idx]; 175 } 176 177 static int ntb_epf_mw_count(struct ntb_dev *ntb, int pidx) 178 { 179 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 180 struct device *dev = ndev->dev; 181 182 if (pidx != NTB_DEF_PEER_IDX) { 183 dev_err(dev, "Unsupported Peer ID %d\n", pidx); 184 return -EINVAL; 185 } 186 187 return ndev->mw_count; 188 } 189 190 static int ntb_epf_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, 191 resource_size_t *addr_align, 192 resource_size_t *size_align, 193 resource_size_t *size_max) 194 { 195 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 196 struct device *dev = ndev->dev; 197 int bar; 198 199 if (pidx != NTB_DEF_PEER_IDX) { 200 dev_err(dev, "Unsupported Peer ID %d\n", pidx); 201 return -EINVAL; 202 } 203 204 bar = ntb_epf_mw_to_bar(ndev, idx); 205 if (bar < 0) 206 return bar; 207 208 if (addr_align) 209 *addr_align = SZ_4K; 210 211 if (size_align) 212 *size_align = 1; 213 214 if (size_max) 215 *size_max = pci_resource_len(ndev->ntb.pdev, bar); 216 217 return 0; 218 } 219 220 static u64 ntb_epf_link_is_up(struct ntb_dev *ntb, 221 enum ntb_speed *speed, 222 enum ntb_width *width) 223 { 224 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 225 u32 status; 226 227 status = readw(ndev->ctrl_reg + NTB_EPF_LINK_STATUS); 228 229 return status & LINK_STATUS_UP; 230 } 231 232 static u32 ntb_epf_spad_read(struct ntb_dev *ntb, int idx) 233 { 234 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 235 struct device *dev = ndev->dev; 236 u32 offset; 237 238 if (idx < 0 || idx >= ndev->spad_count) { 239 dev_err(dev, "READ: Invalid ScratchPad Index %d\n", idx); 240 return 0; 241 } 242 243 offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); 244 offset += (idx << 2); 245 246 return readl(ndev->ctrl_reg + offset); 247 } 248 249 static int ntb_epf_spad_write(struct ntb_dev *ntb, 250 int idx, u32 val) 251 { 252 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 253 struct device *dev = ndev->dev; 254 u32 offset; 255 256 if (idx < 0 || idx >= ndev->spad_count) { 257 dev_err(dev, "WRITE: Invalid ScratchPad Index %d\n", idx); 258 return -EINVAL; 259 } 260 261 offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); 262 offset += (idx << 2); 263 writel(val, ndev->ctrl_reg + offset); 264 265 return 0; 266 } 267 268 static u32 ntb_epf_peer_spad_read(struct ntb_dev *ntb, int pidx, int idx) 269 { 270 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 271 struct device *dev = ndev->dev; 272 u32 offset; 273 274 if (pidx != NTB_DEF_PEER_IDX) { 275 dev_err(dev, "Unsupported Peer ID %d\n", pidx); 276 return -EINVAL; 277 } 278 279 if (idx < 0 || idx >= ndev->spad_count) { 280 dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n", idx); 281 return -EINVAL; 282 } 283 284 offset = (idx << 2); 285 return readl(ndev->peer_spad_reg + offset); 286 } 287 288 static int ntb_epf_peer_spad_write(struct ntb_dev *ntb, int pidx, 289 int idx, u32 val) 290 { 291 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 292 struct device *dev = ndev->dev; 293 u32 offset; 294 295 if (pidx != NTB_DEF_PEER_IDX) { 296 dev_err(dev, "Unsupported Peer ID %d\n", pidx); 297 return -EINVAL; 298 } 299 300 if (idx < 0 || idx >= ndev->spad_count) { 301 dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n", idx); 302 return -EINVAL; 303 } 304 305 offset = (idx << 2); 306 writel(val, ndev->peer_spad_reg + offset); 307 308 return 0; 309 } 310 311 static int ntb_epf_link_enable(struct ntb_dev *ntb, 312 enum ntb_speed max_speed, 313 enum ntb_width max_width) 314 { 315 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 316 struct device *dev = ndev->dev; 317 int ret; 318 319 ret = ntb_epf_send_command(ndev, CMD_LINK_UP, 0); 320 if (ret) { 321 dev_err(dev, "Fail to enable link\n"); 322 return ret; 323 } 324 325 return 0; 326 } 327 328 static int ntb_epf_link_disable(struct ntb_dev *ntb) 329 { 330 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 331 struct device *dev = ndev->dev; 332 int ret; 333 334 ret = ntb_epf_send_command(ndev, CMD_LINK_DOWN, 0); 335 if (ret) { 336 dev_err(dev, "Fail to disable link\n"); 337 return ret; 338 } 339 340 return 0; 341 } 342 343 static irqreturn_t ntb_epf_vec_isr(int irq, void *dev) 344 { 345 struct ntb_epf_irq_ctx *ctx = dev; 346 struct ntb_epf_dev *ndev = ctx->ndev; 347 unsigned int db_vector; 348 unsigned int irq_no = ctx->irq_no; 349 350 if (irq_no == EPF_IRQ_LINK) { 351 ntb_link_event(&ndev->ntb); 352 } else if (irq_no == EPF_IRQ_RESERVED_DB) { 353 dev_warn_ratelimited(ndev->dev, 354 "Unexpected reserved doorbell slot IRQ received\n"); 355 } else { 356 db_vector = irq_no - EPF_IRQ_DB_START; 357 if (ndev->db_count < NTB_EPF_MIN_DB_COUNT || 358 db_vector >= ndev->db_count - 1) { 359 dev_warn_ratelimited(ndev->dev, 360 "Unexpected doorbell vector %u (db_count %u)\n", 361 db_vector, ndev->db_count); 362 return IRQ_HANDLED; 363 } 364 365 atomic64_or(BIT_ULL(db_vector), &ndev->db_val); 366 ntb_db_event(&ndev->ntb, db_vector); 367 } 368 369 return IRQ_HANDLED; 370 } 371 372 static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max) 373 { 374 struct pci_dev *pdev = ndev->ntb.pdev; 375 struct device *dev = ndev->dev; 376 u32 argument = MSIX_ENABLE; 377 int irq; 378 int ret; 379 int i; 380 381 irq = pci_alloc_irq_vectors(pdev, msi_min, msi_max, PCI_IRQ_MSIX); 382 if (irq < 0) { 383 dev_dbg(dev, "Failed to get MSIX interrupts\n"); 384 irq = pci_alloc_irq_vectors(pdev, msi_min, msi_max, 385 PCI_IRQ_MSI); 386 if (irq < 0) { 387 dev_err(dev, "Failed to get MSI interrupts\n"); 388 return irq; 389 } 390 argument &= ~MSIX_ENABLE; 391 } 392 393 ndev->db_count = irq - 1; 394 for (i = 0; i < irq; i++) { 395 ndev->irq_ctx[i].ndev = ndev; 396 ndev->irq_ctx[i].irq_no = i; 397 ret = request_irq(pci_irq_vector(pdev, i), ntb_epf_vec_isr, 398 0, "ntb_epf", &ndev->irq_ctx[i]); 399 if (ret) { 400 dev_err(dev, "Failed to request irq\n"); 401 goto err_free_irq; 402 } 403 } 404 405 ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL, 406 argument | irq); 407 if (ret) { 408 dev_err(dev, "Failed to configure doorbell\n"); 409 goto err_free_irq; 410 } 411 412 return 0; 413 414 err_free_irq: 415 while (i--) 416 free_irq(pci_irq_vector(pdev, i), &ndev->irq_ctx[i]); 417 pci_free_irq_vectors(pdev); 418 419 return ret; 420 } 421 422 static int ntb_epf_peer_mw_count(struct ntb_dev *ntb) 423 { 424 return ntb_ndev(ntb)->mw_count; 425 } 426 427 static int ntb_epf_spad_count(struct ntb_dev *ntb) 428 { 429 return ntb_ndev(ntb)->spad_count; 430 } 431 432 static u64 ntb_epf_db_valid_mask(struct ntb_dev *ntb) 433 { 434 return ntb_ndev(ntb)->db_valid_mask; 435 } 436 437 static int ntb_epf_db_vector_count(struct ntb_dev *ntb) 438 { 439 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 440 unsigned int db_count = ndev->db_count; 441 442 /* 443 * db_count includes an extra skipped slot due to the legacy 444 * doorbell layout. Expose only the real doorbell vectors. 445 */ 446 if (db_count < NTB_EPF_MIN_DB_COUNT) 447 return 0; 448 449 return db_count - 1; 450 } 451 452 static u64 ntb_epf_db_vector_mask(struct ntb_dev *ntb, int db_vector) 453 { 454 int nr_vec; 455 456 /* 457 * db_count includes one skipped slot in the legacy layout. Valid 458 * doorbell vectors are therefore [0 .. (db_count - 2)]. 459 */ 460 nr_vec = ntb_epf_db_vector_count(ntb); 461 if (db_vector < 0 || db_vector >= nr_vec) 462 return 0; 463 464 return BIT_ULL(db_vector); 465 } 466 467 static int ntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 468 { 469 return 0; 470 } 471 472 static int ntb_epf_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, 473 dma_addr_t addr, resource_size_t size) 474 { 475 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 476 struct device *dev = ndev->dev; 477 resource_size_t mw_size; 478 int bar; 479 480 if (pidx != NTB_DEF_PEER_IDX) { 481 dev_err(dev, "Unsupported Peer ID %d\n", pidx); 482 return -EINVAL; 483 } 484 485 bar = ntb_epf_mw_to_bar(ndev, idx); 486 if (bar < 0) 487 return bar; 488 489 mw_size = pci_resource_len(ntb->pdev, bar); 490 491 if (size > mw_size) { 492 dev_err(dev, "Size:%pa is greater than the MW size %pa\n", 493 &size, &mw_size); 494 return -EINVAL; 495 } 496 497 writel(lower_32_bits(addr), ndev->ctrl_reg + NTB_EPF_LOWER_ADDR); 498 writel(upper_32_bits(addr), ndev->ctrl_reg + NTB_EPF_UPPER_ADDR); 499 writel(lower_32_bits(size), ndev->ctrl_reg + NTB_EPF_LOWER_SIZE); 500 writel(upper_32_bits(size), ndev->ctrl_reg + NTB_EPF_UPPER_SIZE); 501 ntb_epf_send_command(ndev, CMD_CONFIGURE_MW, idx); 502 503 return 0; 504 } 505 506 static int ntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx) 507 { 508 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 509 struct device *dev = ndev->dev; 510 int ret = 0; 511 512 ntb_epf_send_command(ndev, CMD_TEARDOWN_MW, idx); 513 if (ret) 514 dev_err(dev, "Failed to teardown memory window\n"); 515 516 return ret; 517 } 518 519 static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 520 phys_addr_t *base, resource_size_t *size) 521 { 522 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 523 u32 offset = 0; 524 int bar; 525 526 if (idx == 0) 527 offset = readl(ndev->ctrl_reg + NTB_EPF_MW1_OFFSET); 528 529 bar = ntb_epf_mw_to_bar(ndev, idx); 530 if (bar < 0) 531 return bar; 532 533 if (base) 534 *base = pci_resource_start(ndev->ntb.pdev, bar) + offset; 535 536 if (size) 537 *size = pci_resource_len(ndev->ntb.pdev, bar) - offset; 538 539 return 0; 540 } 541 542 static int ntb_epf_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 543 { 544 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 545 /* 546 * ffs() returns a 1-based bit index (bit 0 -> 1). 547 * 548 * With slot 0 reserved for link events, DB#0 would naturally map to 549 * slot 1. Historically an extra +1 offset was added, so DB#0 maps to 550 * slot 2 and slot 1 remains unused. Keep this mapping for 551 * backward-compatibility. 552 */ 553 u32 interrupt_num = ffs(db_bits) + 1; 554 struct device *dev = ndev->dev; 555 u32 db_entry_size; 556 u32 db_offset; 557 u32 db_data; 558 559 if (interrupt_num > ndev->db_count) { 560 dev_err(dev, "DB interrupt %d greater than Max Supported %d\n", 561 interrupt_num, ndev->db_count); 562 return -EINVAL; 563 } 564 565 db_entry_size = readl(ndev->ctrl_reg + NTB_EPF_DB_ENTRY_SIZE); 566 567 db_data = readl(ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num)); 568 db_offset = readl(ndev->ctrl_reg + NTB_EPF_DB_OFFSET(interrupt_num)); 569 writel(db_data, ndev->db_reg + (db_entry_size * interrupt_num) + 570 db_offset); 571 572 return 0; 573 } 574 575 static u64 ntb_epf_db_read(struct ntb_dev *ntb) 576 { 577 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 578 579 return atomic64_read(&ndev->db_val); 580 } 581 582 static int ntb_epf_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 583 { 584 return 0; 585 } 586 587 static int ntb_epf_db_clear(struct ntb_dev *ntb, u64 db_bits) 588 { 589 struct ntb_epf_dev *ndev = ntb_ndev(ntb); 590 591 atomic64_and(~db_bits, &ndev->db_val); 592 593 return 0; 594 } 595 596 static const struct ntb_dev_ops ntb_epf_ops = { 597 .mw_count = ntb_epf_mw_count, 598 .spad_count = ntb_epf_spad_count, 599 .peer_mw_count = ntb_epf_peer_mw_count, 600 .db_valid_mask = ntb_epf_db_valid_mask, 601 .db_vector_count = ntb_epf_db_vector_count, 602 .db_vector_mask = ntb_epf_db_vector_mask, 603 .db_set_mask = ntb_epf_db_set_mask, 604 .mw_set_trans = ntb_epf_mw_set_trans, 605 .mw_clear_trans = ntb_epf_mw_clear_trans, 606 .peer_mw_get_addr = ntb_epf_peer_mw_get_addr, 607 .link_enable = ntb_epf_link_enable, 608 .spad_read = ntb_epf_spad_read, 609 .spad_write = ntb_epf_spad_write, 610 .peer_spad_read = ntb_epf_peer_spad_read, 611 .peer_spad_write = ntb_epf_peer_spad_write, 612 .peer_db_set = ntb_epf_peer_db_set, 613 .db_read = ntb_epf_db_read, 614 .mw_get_align = ntb_epf_mw_get_align, 615 .link_is_up = ntb_epf_link_is_up, 616 .db_clear_mask = ntb_epf_db_clear_mask, 617 .db_clear = ntb_epf_db_clear, 618 .link_disable = ntb_epf_link_disable, 619 }; 620 621 static inline void ntb_epf_init_struct(struct ntb_epf_dev *ndev, 622 struct pci_dev *pdev) 623 { 624 ndev->ntb.pdev = pdev; 625 ndev->ntb.topo = NTB_TOPO_NONE; 626 ndev->ntb.ops = &ntb_epf_ops; 627 } 628 629 static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) 630 { 631 struct device *dev = ndev->dev; 632 int ret; 633 634 ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT); 635 if (ndev->mw_count > NTB_EPF_MAX_MW_COUNT) { 636 dev_err(dev, "Unsupported MW count: %u\n", ndev->mw_count); 637 return -EINVAL; 638 } 639 640 /* One Link interrupt and rest doorbell interrupt */ 641 ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1, 642 NTB_EPF_MAX_DB_COUNT + 1); 643 if (ret) { 644 dev_err(dev, "Failed to init ISR\n"); 645 return ret; 646 } 647 648 /* 649 * ndev->db_count includes an extra skipped slot due to the legacy 650 * doorbell layout, hence -1. 651 */ 652 ndev->db_valid_mask = BIT_ULL(ndev->db_count - 1) - 1; 653 ndev->spad_count = readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); 654 655 return 0; 656 } 657 658 static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, 659 struct pci_dev *pdev) 660 { 661 struct device *dev = ndev->dev; 662 size_t spad_sz, spad_off; 663 int ret; 664 665 pci_set_drvdata(pdev, ndev); 666 667 ret = pci_enable_device(pdev); 668 if (ret) { 669 dev_err(dev, "Cannot enable PCI device\n"); 670 goto err_pci_enable; 671 } 672 673 ret = pci_request_regions(pdev, "ntb"); 674 if (ret) { 675 dev_err(dev, "Cannot obtain PCI resources\n"); 676 goto err_pci_regions; 677 } 678 679 pci_set_master(pdev); 680 681 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 682 if (ret) { 683 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 684 if (ret) { 685 dev_err(dev, "Cannot set DMA mask\n"); 686 goto err_pci_regions; 687 } 688 dev_warn(&pdev->dev, "Cannot DMA highmem\n"); 689 } 690 691 ndev->ctrl_reg = pci_iomap(pdev, ndev->barno_map[BAR_CONFIG], 0); 692 if (!ndev->ctrl_reg) { 693 ret = -EIO; 694 goto err_pci_regions; 695 } 696 697 if (ndev->barno_map[BAR_PEER_SPAD] != ndev->barno_map[BAR_CONFIG]) { 698 ndev->peer_spad_reg = pci_iomap(pdev, 699 ndev->barno_map[BAR_PEER_SPAD], 0); 700 if (!ndev->peer_spad_reg) { 701 ret = -EIO; 702 goto err_pci_regions; 703 } 704 } else { 705 spad_sz = 4 * readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); 706 spad_off = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); 707 ndev->peer_spad_reg = ndev->ctrl_reg + spad_off + spad_sz; 708 } 709 710 ndev->db_reg = pci_iomap(pdev, ndev->barno_map[BAR_DB], 0); 711 if (!ndev->db_reg) { 712 ret = -EIO; 713 goto err_pci_regions; 714 } 715 716 return 0; 717 718 err_pci_regions: 719 pci_disable_device(pdev); 720 721 err_pci_enable: 722 pci_set_drvdata(pdev, NULL); 723 724 return ret; 725 } 726 727 static void ntb_epf_deinit_pci(struct ntb_epf_dev *ndev) 728 { 729 struct pci_dev *pdev = ndev->ntb.pdev; 730 731 pci_iounmap(pdev, ndev->ctrl_reg); 732 pci_iounmap(pdev, ndev->peer_spad_reg); 733 pci_iounmap(pdev, ndev->db_reg); 734 735 pci_release_regions(pdev); 736 pci_disable_device(pdev); 737 pci_set_drvdata(pdev, NULL); 738 } 739 740 static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev) 741 { 742 struct pci_dev *pdev = ndev->ntb.pdev; 743 int i; 744 745 ntb_epf_send_command(ndev, CMD_TEARDOWN_DOORBELL, ndev->db_count + 1); 746 747 for (i = 0; i < ndev->db_count + 1; i++) 748 free_irq(pci_irq_vector(pdev, i), &ndev->irq_ctx[i]); 749 pci_free_irq_vectors(pdev); 750 } 751 752 static int ntb_epf_pci_probe(struct pci_dev *pdev, 753 const struct pci_device_id *id) 754 { 755 struct device *dev = &pdev->dev; 756 struct ntb_epf_dev *ndev; 757 int ret; 758 759 if (pci_is_bridge(pdev)) 760 return -ENODEV; 761 762 ndev = devm_kzalloc(dev, sizeof(*ndev), GFP_KERNEL); 763 if (!ndev) 764 return -ENOMEM; 765 766 ndev->barno_map = (const enum pci_barno *)id->driver_data; 767 if (!ndev->barno_map) 768 return -EINVAL; 769 770 ndev->dev = dev; 771 772 ntb_epf_init_struct(ndev, pdev); 773 mutex_init(&ndev->cmd_lock); 774 775 ret = ntb_epf_init_pci(ndev, pdev); 776 if (ret) { 777 dev_err(dev, "Failed to init PCI\n"); 778 return ret; 779 } 780 781 ret = ntb_epf_init_dev(ndev); 782 if (ret) { 783 dev_err(dev, "Failed to init device\n"); 784 goto err_init_dev; 785 } 786 787 ret = ntb_register_device(&ndev->ntb); 788 if (ret) { 789 dev_err(dev, "Failed to register NTB device\n"); 790 goto err_register_dev; 791 } 792 793 return 0; 794 795 err_register_dev: 796 ntb_epf_cleanup_isr(ndev); 797 798 err_init_dev: 799 ntb_epf_deinit_pci(ndev); 800 801 return ret; 802 } 803 804 static void ntb_epf_pci_remove(struct pci_dev *pdev) 805 { 806 struct ntb_epf_dev *ndev = pci_get_drvdata(pdev); 807 808 ntb_unregister_device(&ndev->ntb); 809 ntb_epf_cleanup_isr(ndev); 810 ntb_epf_deinit_pci(ndev); 811 } 812 813 static const enum pci_barno j721e_map[NTB_BAR_NUM] = { 814 [BAR_CONFIG] = BAR_0, 815 [BAR_PEER_SPAD] = BAR_1, 816 [BAR_DB] = BAR_2, 817 [BAR_MW1] = BAR_2, 818 [BAR_MW2] = BAR_3, 819 [BAR_MW3] = BAR_4, 820 [BAR_MW4] = BAR_5 821 }; 822 823 static const enum pci_barno mx8_map[NTB_BAR_NUM] = { 824 [BAR_CONFIG] = BAR_0, 825 [BAR_PEER_SPAD] = BAR_0, 826 [BAR_DB] = BAR_2, 827 [BAR_MW1] = BAR_4, 828 [BAR_MW2] = BAR_5, 829 [BAR_MW3] = NO_BAR, 830 [BAR_MW4] = NO_BAR 831 }; 832 833 static const enum pci_barno rcar_barno[NTB_BAR_NUM] = { 834 [BAR_CONFIG] = BAR_0, 835 [BAR_PEER_SPAD] = BAR_0, 836 [BAR_DB] = BAR_4, 837 [BAR_MW1] = BAR_2, 838 [BAR_MW2] = NO_BAR, 839 [BAR_MW3] = NO_BAR, 840 [BAR_MW4] = NO_BAR, 841 }; 842 843 static const struct pci_device_id ntb_epf_pci_tbl[] = { 844 { 845 PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E), 846 .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, 847 .driver_data = (kernel_ulong_t)j721e_map, 848 }, 849 { 850 PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x0809), 851 .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, 852 .driver_data = (kernel_ulong_t)mx8_map, 853 }, 854 { 855 PCI_DEVICE(PCI_VENDOR_ID_RENESAS, 0x0030), 856 .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, 857 .driver_data = (kernel_ulong_t)rcar_barno, 858 }, 859 { }, 860 }; 861 862 static struct pci_driver ntb_epf_pci_driver = { 863 .name = KBUILD_MODNAME, 864 .id_table = ntb_epf_pci_tbl, 865 .probe = ntb_epf_pci_probe, 866 .remove = ntb_epf_pci_remove, 867 }; 868 module_pci_driver(ntb_epf_pci_driver); 869 870 MODULE_DESCRIPTION("PCI ENDPOINT NTB HOST DRIVER"); 871 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 872 MODULE_LICENSE("GPL v2"); 873