1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * BSD LICENSE 14 * 15 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * * Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * * Redistributions in binary form must reproduce the above copy 24 * notice, this list of conditions and the following disclaimer in 25 * the documentation and/or other materials provided with the 26 * distribution. 27 * * Neither the name of AMD Corporation nor the names of its 28 * contributors may be used to endorse or promote products derived 29 * from this software without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * AMD PCIe NTB Linux driver 44 * 45 * Contact Information: 46 * Xiangliang Yu <Xiangliang.Yu@amd.com> 47 */ 48 49 #include <linux/debugfs.h> 50 #include <linux/delay.h> 51 #include <linux/init.h> 52 #include <linux/interrupt.h> 53 #include <linux/module.h> 54 #include <linux/acpi.h> 55 #include <linux/pci.h> 56 #include <linux/random.h> 57 #include <linux/slab.h> 58 #include <linux/ntb.h> 59 60 #include "ntb_hw_amd.h" 61 62 #define NTB_NAME "ntb_hw_amd" 63 #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" 64 #define NTB_VER "1.0" 65 66 MODULE_DESCRIPTION(NTB_DESC); 67 MODULE_VERSION(NTB_VER); 68 MODULE_LICENSE("Dual BSD/GPL"); 69 MODULE_AUTHOR("AMD Inc."); 70 71 static const struct file_operations amd_ntb_debugfs_info; 72 static struct dentry *debugfs_dir; 73 74 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) 75 { 76 if (idx < 0 || idx > ndev->mw_count) 77 return -EINVAL; 78 79 return 1 << idx; 80 } 81 82 static int amd_ntb_mw_count(struct ntb_dev *ntb) 83 { 84 return ntb_ndev(ntb)->mw_count; 85 } 86 87 static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx, 88 phys_addr_t *base, 89 resource_size_t *size, 90 resource_size_t *align, 91 resource_size_t *align_size) 92 { 93 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 94 int bar; 95 96 bar = ndev_mw_to_bar(ndev, idx); 97 if (bar < 0) 98 return bar; 99 100 if (base) 101 *base = pci_resource_start(ndev->ntb.pdev, bar); 102 103 if (size) 104 *size = pci_resource_len(ndev->ntb.pdev, bar); 105 106 if (align) 107 *align = SZ_4K; 108 109 if (align_size) 110 *align_size = 1; 111 112 return 0; 113 } 114 115 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx, 116 dma_addr_t addr, resource_size_t size) 117 { 118 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 119 unsigned long xlat_reg, limit_reg = 0; 120 resource_size_t mw_size; 121 void __iomem *mmio, *peer_mmio; 122 u64 base_addr, limit, reg_val; 123 int bar; 124 125 bar = ndev_mw_to_bar(ndev, idx); 126 if (bar < 0) 127 return bar; 128 129 mw_size = pci_resource_len(ndev->ntb.pdev, bar); 130 131 /* make sure the range fits in the usable mw size */ 132 if (size > mw_size) 133 return -EINVAL; 134 135 mmio = ndev->self_mmio; 136 peer_mmio = ndev->peer_mmio; 137 138 base_addr = pci_resource_start(ndev->ntb.pdev, bar); 139 140 if (bar != 1) { 141 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 3); 142 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 3); 143 144 /* Set the limit if supported */ 145 limit = base_addr + size; 146 147 /* set and verify setting the translation address */ 148 write64(addr, peer_mmio + xlat_reg); 149 reg_val = read64(peer_mmio + xlat_reg); 150 if (reg_val != addr) { 151 write64(0, peer_mmio + xlat_reg); 152 return -EIO; 153 } 154 155 /* set and verify setting the limit */ 156 write64(limit, mmio + limit_reg); 157 reg_val = read64(mmio + limit_reg); 158 if (reg_val != limit) { 159 write64(base_addr, mmio + limit_reg); 160 write64(0, peer_mmio + xlat_reg); 161 return -EIO; 162 } 163 } else { 164 xlat_reg = AMD_BAR1XLAT_OFFSET; 165 limit_reg = AMD_BAR1LMT_OFFSET; 166 167 /* split bar addr range must all be 32 bit */ 168 if (addr & (~0ull << 32)) 169 return -EINVAL; 170 if ((addr + size) & (~0ull << 32)) 171 return -EINVAL; 172 173 /* Set the limit if supported */ 174 limit = base_addr + size; 175 176 /* set and verify setting the translation address */ 177 write64(addr, peer_mmio + xlat_reg); 178 reg_val = read64(peer_mmio + xlat_reg); 179 if (reg_val != addr) { 180 write64(0, peer_mmio + xlat_reg); 181 return -EIO; 182 } 183 184 /* set and verify setting the limit */ 185 writel(limit, mmio + limit_reg); 186 reg_val = readl(mmio + limit_reg); 187 if (reg_val != limit) { 188 writel(base_addr, mmio + limit_reg); 189 writel(0, peer_mmio + xlat_reg); 190 return -EIO; 191 } 192 } 193 194 return 0; 195 } 196 197 static int amd_link_is_up(struct amd_ntb_dev *ndev) 198 { 199 if (!ndev->peer_sta) 200 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta); 201 202 /* If peer_sta is reset or D0 event, the ISR has 203 * started a timer to check link status of hardware. 204 * So here just clear status bit. And if peer_sta is 205 * D3 or PME_TO, D0/reset event will be happened when 206 * system wakeup/poweron, so do nothing here. 207 */ 208 if (ndev->peer_sta & AMD_PEER_RESET_EVENT) 209 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT; 210 else if (ndev->peer_sta & AMD_PEER_D0_EVENT) 211 ndev->peer_sta = 0; 212 213 return 0; 214 } 215 216 static int amd_ntb_link_is_up(struct ntb_dev *ntb, 217 enum ntb_speed *speed, 218 enum ntb_width *width) 219 { 220 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 221 int ret = 0; 222 223 if (amd_link_is_up(ndev)) { 224 if (speed) 225 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 226 if (width) 227 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 228 229 dev_dbg(ndev_dev(ndev), "link is up.\n"); 230 231 ret = 1; 232 } else { 233 if (speed) 234 *speed = NTB_SPEED_NONE; 235 if (width) 236 *width = NTB_WIDTH_NONE; 237 238 dev_dbg(ndev_dev(ndev), "link is down.\n"); 239 } 240 241 return ret; 242 } 243 244 static int amd_ntb_link_enable(struct ntb_dev *ntb, 245 enum ntb_speed max_speed, 246 enum ntb_width max_width) 247 { 248 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 249 void __iomem *mmio = ndev->self_mmio; 250 u32 ntb_ctl; 251 252 /* Enable event interrupt */ 253 ndev->int_mask &= ~AMD_EVENT_INTMASK; 254 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 255 256 if (ndev->ntb.topo == NTB_TOPO_SEC) 257 return -EINVAL; 258 dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 259 260 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 261 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); 262 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 263 264 return 0; 265 } 266 267 static int amd_ntb_link_disable(struct ntb_dev *ntb) 268 { 269 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 270 void __iomem *mmio = ndev->self_mmio; 271 u32 ntb_ctl; 272 273 /* Disable event interrupt */ 274 ndev->int_mask |= AMD_EVENT_INTMASK; 275 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 276 277 if (ndev->ntb.topo == NTB_TOPO_SEC) 278 return -EINVAL; 279 dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 280 281 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 282 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); 283 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 284 285 return 0; 286 } 287 288 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) 289 { 290 return ntb_ndev(ntb)->db_valid_mask; 291 } 292 293 static int amd_ntb_db_vector_count(struct ntb_dev *ntb) 294 { 295 return ntb_ndev(ntb)->db_count; 296 } 297 298 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 299 { 300 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 301 302 if (db_vector < 0 || db_vector > ndev->db_count) 303 return 0; 304 305 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector); 306 } 307 308 static u64 amd_ntb_db_read(struct ntb_dev *ntb) 309 { 310 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 311 void __iomem *mmio = ndev->self_mmio; 312 313 return (u64)readw(mmio + AMD_DBSTAT_OFFSET); 314 } 315 316 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 317 { 318 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 319 void __iomem *mmio = ndev->self_mmio; 320 321 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET); 322 323 return 0; 324 } 325 326 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 327 { 328 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 329 void __iomem *mmio = ndev->self_mmio; 330 unsigned long flags; 331 332 if (db_bits & ~ndev->db_valid_mask) 333 return -EINVAL; 334 335 spin_lock_irqsave(&ndev->db_mask_lock, flags); 336 ndev->db_mask |= db_bits; 337 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 338 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 339 340 return 0; 341 } 342 343 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 344 { 345 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 346 void __iomem *mmio = ndev->self_mmio; 347 unsigned long flags; 348 349 if (db_bits & ~ndev->db_valid_mask) 350 return -EINVAL; 351 352 spin_lock_irqsave(&ndev->db_mask_lock, flags); 353 ndev->db_mask &= ~db_bits; 354 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 355 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 356 357 return 0; 358 } 359 360 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 361 { 362 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 363 void __iomem *mmio = ndev->self_mmio; 364 365 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET); 366 367 return 0; 368 } 369 370 static int amd_ntb_spad_count(struct ntb_dev *ntb) 371 { 372 return ntb_ndev(ntb)->spad_count; 373 } 374 375 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) 376 { 377 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 378 void __iomem *mmio = ndev->self_mmio; 379 u32 offset; 380 381 if (idx < 0 || idx >= ndev->spad_count) 382 return 0; 383 384 offset = ndev->self_spad + (idx << 2); 385 return readl(mmio + AMD_SPAD_OFFSET + offset); 386 } 387 388 static int amd_ntb_spad_write(struct ntb_dev *ntb, 389 int idx, u32 val) 390 { 391 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 392 void __iomem *mmio = ndev->self_mmio; 393 u32 offset; 394 395 if (idx < 0 || idx >= ndev->spad_count) 396 return -EINVAL; 397 398 offset = ndev->self_spad + (idx << 2); 399 writel(val, mmio + AMD_SPAD_OFFSET + offset); 400 401 return 0; 402 } 403 404 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx) 405 { 406 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 407 void __iomem *mmio = ndev->self_mmio; 408 u32 offset; 409 410 if (idx < 0 || idx >= ndev->spad_count) 411 return -EINVAL; 412 413 offset = ndev->peer_spad + (idx << 2); 414 return readl(mmio + AMD_SPAD_OFFSET + offset); 415 } 416 417 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, 418 int idx, u32 val) 419 { 420 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 421 void __iomem *mmio = ndev->self_mmio; 422 u32 offset; 423 424 if (idx < 0 || idx >= ndev->spad_count) 425 return -EINVAL; 426 427 offset = ndev->peer_spad + (idx << 2); 428 writel(val, mmio + AMD_SPAD_OFFSET + offset); 429 430 return 0; 431 } 432 433 static const struct ntb_dev_ops amd_ntb_ops = { 434 .mw_count = amd_ntb_mw_count, 435 .mw_get_range = amd_ntb_mw_get_range, 436 .mw_set_trans = amd_ntb_mw_set_trans, 437 .link_is_up = amd_ntb_link_is_up, 438 .link_enable = amd_ntb_link_enable, 439 .link_disable = amd_ntb_link_disable, 440 .db_valid_mask = amd_ntb_db_valid_mask, 441 .db_vector_count = amd_ntb_db_vector_count, 442 .db_vector_mask = amd_ntb_db_vector_mask, 443 .db_read = amd_ntb_db_read, 444 .db_clear = amd_ntb_db_clear, 445 .db_set_mask = amd_ntb_db_set_mask, 446 .db_clear_mask = amd_ntb_db_clear_mask, 447 .peer_db_set = amd_ntb_peer_db_set, 448 .spad_count = amd_ntb_spad_count, 449 .spad_read = amd_ntb_spad_read, 450 .spad_write = amd_ntb_spad_write, 451 .peer_spad_read = amd_ntb_peer_spad_read, 452 .peer_spad_write = amd_ntb_peer_spad_write, 453 }; 454 455 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) 456 { 457 void __iomem *mmio = ndev->self_mmio; 458 int reg; 459 460 reg = readl(mmio + AMD_SMUACK_OFFSET); 461 reg |= bit; 462 writel(reg, mmio + AMD_SMUACK_OFFSET); 463 464 ndev->peer_sta |= bit; 465 } 466 467 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) 468 { 469 void __iomem *mmio = ndev->self_mmio; 470 u32 status; 471 472 status = readl(mmio + AMD_INTSTAT_OFFSET); 473 if (!(status & AMD_EVENT_INTMASK)) 474 return; 475 476 dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec); 477 478 status &= AMD_EVENT_INTMASK; 479 switch (status) { 480 case AMD_PEER_FLUSH_EVENT: 481 dev_info(ndev_dev(ndev), "Flush is done.\n"); 482 break; 483 case AMD_PEER_RESET_EVENT: 484 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT); 485 486 /* link down first */ 487 ntb_link_event(&ndev->ntb); 488 /* polling peer status */ 489 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 490 491 break; 492 case AMD_PEER_D3_EVENT: 493 case AMD_PEER_PMETO_EVENT: 494 amd_ack_smu(ndev, status); 495 496 /* link down */ 497 ntb_link_event(&ndev->ntb); 498 499 break; 500 case AMD_PEER_D0_EVENT: 501 mmio = ndev->peer_mmio; 502 status = readl(mmio + AMD_PMESTAT_OFFSET); 503 /* check if this is WAKEUP event */ 504 if (status & 0x1) 505 dev_info(ndev_dev(ndev), "Wakeup is done.\n"); 506 507 amd_ack_smu(ndev, AMD_PEER_D0_EVENT); 508 509 /* start a timer to poll link status */ 510 schedule_delayed_work(&ndev->hb_timer, 511 AMD_LINK_HB_TIMEOUT); 512 break; 513 default: 514 dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status); 515 break; 516 } 517 } 518 519 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) 520 { 521 dev_dbg(ndev_dev(ndev), "vec %d\n", vec); 522 523 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) 524 amd_handle_event(ndev, vec); 525 526 if (vec < AMD_DB_CNT) 527 ntb_db_event(&ndev->ntb, vec); 528 529 return IRQ_HANDLED; 530 } 531 532 static irqreturn_t ndev_vec_isr(int irq, void *dev) 533 { 534 struct amd_ntb_vec *nvec = dev; 535 536 return ndev_interrupt(nvec->ndev, nvec->num); 537 } 538 539 static irqreturn_t ndev_irq_isr(int irq, void *dev) 540 { 541 struct amd_ntb_dev *ndev = dev; 542 543 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq); 544 } 545 546 static int ndev_init_isr(struct amd_ntb_dev *ndev, 547 int msix_min, int msix_max) 548 { 549 struct pci_dev *pdev; 550 int rc, i, msix_count, node; 551 552 pdev = ndev_pdev(ndev); 553 554 node = dev_to_node(&pdev->dev); 555 556 ndev->db_mask = ndev->db_valid_mask; 557 558 /* Try to set up msix irq */ 559 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec), 560 GFP_KERNEL, node); 561 if (!ndev->vec) 562 goto err_msix_vec_alloc; 563 564 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix), 565 GFP_KERNEL, node); 566 if (!ndev->msix) 567 goto err_msix_alloc; 568 569 for (i = 0; i < msix_max; ++i) 570 ndev->msix[i].entry = i; 571 572 msix_count = pci_enable_msix_range(pdev, ndev->msix, 573 msix_min, msix_max); 574 if (msix_count < 0) 575 goto err_msix_enable; 576 577 /* NOTE: Disable MSIX if msix count is less than 16 because of 578 * hardware limitation. 579 */ 580 if (msix_count < msix_min) { 581 pci_disable_msix(pdev); 582 goto err_msix_enable; 583 } 584 585 for (i = 0; i < msix_count; ++i) { 586 ndev->vec[i].ndev = ndev; 587 ndev->vec[i].num = i; 588 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 589 "ndev_vec_isr", &ndev->vec[i]); 590 if (rc) 591 goto err_msix_request; 592 } 593 594 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n"); 595 ndev->db_count = msix_min; 596 ndev->msix_vec_count = msix_max; 597 return 0; 598 599 err_msix_request: 600 while (i-- > 0) 601 free_irq(ndev->msix[i].vector, ndev); 602 pci_disable_msix(pdev); 603 err_msix_enable: 604 kfree(ndev->msix); 605 err_msix_alloc: 606 kfree(ndev->vec); 607 err_msix_vec_alloc: 608 ndev->msix = NULL; 609 ndev->vec = NULL; 610 611 /* Try to set up msi irq */ 612 rc = pci_enable_msi(pdev); 613 if (rc) 614 goto err_msi_enable; 615 616 rc = request_irq(pdev->irq, ndev_irq_isr, 0, 617 "ndev_irq_isr", ndev); 618 if (rc) 619 goto err_msi_request; 620 621 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n"); 622 ndev->db_count = 1; 623 ndev->msix_vec_count = 1; 624 return 0; 625 626 err_msi_request: 627 pci_disable_msi(pdev); 628 err_msi_enable: 629 630 /* Try to set up intx irq */ 631 pci_intx(pdev, 1); 632 633 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 634 "ndev_irq_isr", ndev); 635 if (rc) 636 goto err_intx_request; 637 638 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n"); 639 ndev->db_count = 1; 640 ndev->msix_vec_count = 1; 641 return 0; 642 643 err_intx_request: 644 return rc; 645 } 646 647 static void ndev_deinit_isr(struct amd_ntb_dev *ndev) 648 { 649 struct pci_dev *pdev; 650 void __iomem *mmio = ndev->self_mmio; 651 int i; 652 653 pdev = ndev_pdev(ndev); 654 655 /* Mask all doorbell interrupts */ 656 ndev->db_mask = ndev->db_valid_mask; 657 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 658 659 if (ndev->msix) { 660 i = ndev->msix_vec_count; 661 while (i--) 662 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 663 pci_disable_msix(pdev); 664 kfree(ndev->msix); 665 kfree(ndev->vec); 666 } else { 667 free_irq(pdev->irq, ndev); 668 if (pci_dev_msi_enabled(pdev)) 669 pci_disable_msi(pdev); 670 else 671 pci_intx(pdev, 0); 672 } 673 } 674 675 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 676 size_t count, loff_t *offp) 677 { 678 struct amd_ntb_dev *ndev; 679 void __iomem *mmio; 680 char *buf; 681 size_t buf_size; 682 ssize_t ret, off; 683 union { u64 v64; u32 v32; u16 v16; } u; 684 685 ndev = filp->private_data; 686 mmio = ndev->self_mmio; 687 688 buf_size = min(count, 0x800ul); 689 690 buf = kmalloc(buf_size, GFP_KERNEL); 691 if (!buf) 692 return -ENOMEM; 693 694 off = 0; 695 696 off += scnprintf(buf + off, buf_size - off, 697 "NTB Device Information:\n"); 698 699 off += scnprintf(buf + off, buf_size - off, 700 "Connection Topology -\t%s\n", 701 ntb_topo_string(ndev->ntb.topo)); 702 703 off += scnprintf(buf + off, buf_size - off, 704 "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 705 706 if (!amd_link_is_up(ndev)) { 707 off += scnprintf(buf + off, buf_size - off, 708 "Link Status -\t\tDown\n"); 709 } else { 710 off += scnprintf(buf + off, buf_size - off, 711 "Link Status -\t\tUp\n"); 712 off += scnprintf(buf + off, buf_size - off, 713 "Link Speed -\t\tPCI-E Gen %u\n", 714 NTB_LNK_STA_SPEED(ndev->lnk_sta)); 715 off += scnprintf(buf + off, buf_size - off, 716 "Link Width -\t\tx%u\n", 717 NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 718 } 719 720 off += scnprintf(buf + off, buf_size - off, 721 "Memory Window Count -\t%u\n", ndev->mw_count); 722 off += scnprintf(buf + off, buf_size - off, 723 "Scratchpad Count -\t%u\n", ndev->spad_count); 724 off += scnprintf(buf + off, buf_size - off, 725 "Doorbell Count -\t%u\n", ndev->db_count); 726 off += scnprintf(buf + off, buf_size - off, 727 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count); 728 729 off += scnprintf(buf + off, buf_size - off, 730 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 731 732 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET); 733 off += scnprintf(buf + off, buf_size - off, 734 "Doorbell Mask -\t\t\t%#06x\n", u.v32); 735 736 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET); 737 off += scnprintf(buf + off, buf_size - off, 738 "Doorbell Bell -\t\t\t%#06x\n", u.v32); 739 740 off += scnprintf(buf + off, buf_size - off, 741 "\nNTB Incoming XLAT:\n"); 742 743 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET); 744 off += scnprintf(buf + off, buf_size - off, 745 "XLAT1 -\t\t%#018llx\n", u.v64); 746 747 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET); 748 off += scnprintf(buf + off, buf_size - off, 749 "XLAT23 -\t\t%#018llx\n", u.v64); 750 751 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET); 752 off += scnprintf(buf + off, buf_size - off, 753 "XLAT45 -\t\t%#018llx\n", u.v64); 754 755 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET); 756 off += scnprintf(buf + off, buf_size - off, 757 "LMT1 -\t\t\t%#06x\n", u.v32); 758 759 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET); 760 off += scnprintf(buf + off, buf_size - off, 761 "LMT23 -\t\t\t%#018llx\n", u.v64); 762 763 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET); 764 off += scnprintf(buf + off, buf_size - off, 765 "LMT45 -\t\t\t%#018llx\n", u.v64); 766 767 ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 768 kfree(buf); 769 return ret; 770 } 771 772 static void ndev_init_debugfs(struct amd_ntb_dev *ndev) 773 { 774 if (!debugfs_dir) { 775 ndev->debugfs_dir = NULL; 776 ndev->debugfs_info = NULL; 777 } else { 778 ndev->debugfs_dir = 779 debugfs_create_dir(ndev_name(ndev), debugfs_dir); 780 if (!ndev->debugfs_dir) 781 ndev->debugfs_info = NULL; 782 else 783 ndev->debugfs_info = 784 debugfs_create_file("info", S_IRUSR, 785 ndev->debugfs_dir, ndev, 786 &amd_ntb_debugfs_info); 787 } 788 } 789 790 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) 791 { 792 debugfs_remove_recursive(ndev->debugfs_dir); 793 } 794 795 static inline void ndev_init_struct(struct amd_ntb_dev *ndev, 796 struct pci_dev *pdev) 797 { 798 ndev->ntb.pdev = pdev; 799 ndev->ntb.topo = NTB_TOPO_NONE; 800 ndev->ntb.ops = &amd_ntb_ops; 801 ndev->int_mask = AMD_EVENT_INTMASK; 802 spin_lock_init(&ndev->db_mask_lock); 803 } 804 805 static int amd_poll_link(struct amd_ntb_dev *ndev) 806 { 807 void __iomem *mmio = ndev->peer_mmio; 808 u32 reg, stat; 809 int rc; 810 811 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 812 reg &= NTB_LIN_STA_ACTIVE_BIT; 813 814 dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg); 815 816 if (reg == ndev->cntl_sta) 817 return 0; 818 819 ndev->cntl_sta = reg; 820 821 rc = pci_read_config_dword(ndev->ntb.pdev, 822 AMD_LINK_STATUS_OFFSET, &stat); 823 if (rc) 824 return 0; 825 ndev->lnk_sta = stat; 826 827 return 1; 828 } 829 830 static void amd_link_hb(struct work_struct *work) 831 { 832 struct amd_ntb_dev *ndev = hb_ndev(work); 833 834 if (amd_poll_link(ndev)) 835 ntb_link_event(&ndev->ntb); 836 837 if (!amd_link_is_up(ndev)) 838 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 839 } 840 841 static int amd_init_isr(struct amd_ntb_dev *ndev) 842 { 843 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); 844 } 845 846 static void amd_init_side_info(struct amd_ntb_dev *ndev) 847 { 848 void __iomem *mmio = ndev->self_mmio; 849 unsigned int reg; 850 851 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 852 if (!(reg & AMD_SIDE_READY)) { 853 reg |= AMD_SIDE_READY; 854 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 855 } 856 } 857 858 static void amd_deinit_side_info(struct amd_ntb_dev *ndev) 859 { 860 void __iomem *mmio = ndev->self_mmio; 861 unsigned int reg; 862 863 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 864 if (reg & AMD_SIDE_READY) { 865 reg &= ~AMD_SIDE_READY; 866 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 867 readl(mmio + AMD_SIDEINFO_OFFSET); 868 } 869 } 870 871 static int amd_init_ntb(struct amd_ntb_dev *ndev) 872 { 873 void __iomem *mmio = ndev->self_mmio; 874 875 ndev->mw_count = AMD_MW_CNT; 876 ndev->spad_count = AMD_SPADS_CNT; 877 ndev->db_count = AMD_DB_CNT; 878 879 switch (ndev->ntb.topo) { 880 case NTB_TOPO_PRI: 881 case NTB_TOPO_SEC: 882 ndev->spad_count >>= 1; 883 if (ndev->ntb.topo == NTB_TOPO_PRI) { 884 ndev->self_spad = 0; 885 ndev->peer_spad = 0x20; 886 } else { 887 ndev->self_spad = 0x20; 888 ndev->peer_spad = 0; 889 } 890 891 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); 892 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 893 894 break; 895 default: 896 dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n"); 897 return -EINVAL; 898 } 899 900 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 901 902 /* Mask event interrupts */ 903 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 904 905 return 0; 906 } 907 908 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) 909 { 910 void __iomem *mmio = ndev->self_mmio; 911 u32 info; 912 913 info = readl(mmio + AMD_SIDEINFO_OFFSET); 914 if (info & AMD_SIDE_MASK) 915 return NTB_TOPO_SEC; 916 else 917 return NTB_TOPO_PRI; 918 } 919 920 static int amd_init_dev(struct amd_ntb_dev *ndev) 921 { 922 struct pci_dev *pdev; 923 int rc = 0; 924 925 pdev = ndev_pdev(ndev); 926 927 ndev->ntb.topo = amd_get_topo(ndev); 928 dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n", 929 ntb_topo_string(ndev->ntb.topo)); 930 931 rc = amd_init_ntb(ndev); 932 if (rc) 933 return rc; 934 935 rc = amd_init_isr(ndev); 936 if (rc) { 937 dev_err(ndev_dev(ndev), "fail to init isr.\n"); 938 return rc; 939 } 940 941 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 942 943 return 0; 944 } 945 946 static void amd_deinit_dev(struct amd_ntb_dev *ndev) 947 { 948 cancel_delayed_work_sync(&ndev->hb_timer); 949 950 ndev_deinit_isr(ndev); 951 } 952 953 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, 954 struct pci_dev *pdev) 955 { 956 int rc; 957 958 pci_set_drvdata(pdev, ndev); 959 960 rc = pci_enable_device(pdev); 961 if (rc) 962 goto err_pci_enable; 963 964 rc = pci_request_regions(pdev, NTB_NAME); 965 if (rc) 966 goto err_pci_regions; 967 968 pci_set_master(pdev); 969 970 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 971 if (rc) { 972 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 973 if (rc) 974 goto err_dma_mask; 975 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n"); 976 } 977 978 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 979 if (rc) { 980 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 981 if (rc) 982 goto err_dma_mask; 983 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n"); 984 } 985 986 ndev->self_mmio = pci_iomap(pdev, 0, 0); 987 if (!ndev->self_mmio) { 988 rc = -EIO; 989 goto err_dma_mask; 990 } 991 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; 992 993 return 0; 994 995 err_dma_mask: 996 pci_clear_master(pdev); 997 err_pci_regions: 998 pci_disable_device(pdev); 999 err_pci_enable: 1000 pci_set_drvdata(pdev, NULL); 1001 return rc; 1002 } 1003 1004 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) 1005 { 1006 struct pci_dev *pdev = ndev_pdev(ndev); 1007 1008 pci_iounmap(pdev, ndev->self_mmio); 1009 1010 pci_clear_master(pdev); 1011 pci_release_regions(pdev); 1012 pci_disable_device(pdev); 1013 pci_set_drvdata(pdev, NULL); 1014 } 1015 1016 static int amd_ntb_pci_probe(struct pci_dev *pdev, 1017 const struct pci_device_id *id) 1018 { 1019 struct amd_ntb_dev *ndev; 1020 int rc, node; 1021 1022 node = dev_to_node(&pdev->dev); 1023 1024 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1025 if (!ndev) { 1026 rc = -ENOMEM; 1027 goto err_ndev; 1028 } 1029 1030 ndev_init_struct(ndev, pdev); 1031 1032 rc = amd_ntb_init_pci(ndev, pdev); 1033 if (rc) 1034 goto err_init_pci; 1035 1036 rc = amd_init_dev(ndev); 1037 if (rc) 1038 goto err_init_dev; 1039 1040 /* write side info */ 1041 amd_init_side_info(ndev); 1042 1043 amd_poll_link(ndev); 1044 1045 ndev_init_debugfs(ndev); 1046 1047 rc = ntb_register_device(&ndev->ntb); 1048 if (rc) 1049 goto err_register; 1050 1051 dev_info(&pdev->dev, "NTB device registered.\n"); 1052 1053 return 0; 1054 1055 err_register: 1056 ndev_deinit_debugfs(ndev); 1057 amd_deinit_dev(ndev); 1058 err_init_dev: 1059 amd_ntb_deinit_pci(ndev); 1060 err_init_pci: 1061 kfree(ndev); 1062 err_ndev: 1063 return rc; 1064 } 1065 1066 static void amd_ntb_pci_remove(struct pci_dev *pdev) 1067 { 1068 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); 1069 1070 ntb_unregister_device(&ndev->ntb); 1071 ndev_deinit_debugfs(ndev); 1072 amd_deinit_side_info(ndev); 1073 amd_deinit_dev(ndev); 1074 amd_ntb_deinit_pci(ndev); 1075 kfree(ndev); 1076 } 1077 1078 static const struct file_operations amd_ntb_debugfs_info = { 1079 .owner = THIS_MODULE, 1080 .open = simple_open, 1081 .read = ndev_debugfs_read, 1082 }; 1083 1084 static const struct pci_device_id amd_ntb_pci_tbl[] = { 1085 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)}, 1086 {0} 1087 }; 1088 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); 1089 1090 static struct pci_driver amd_ntb_pci_driver = { 1091 .name = KBUILD_MODNAME, 1092 .id_table = amd_ntb_pci_tbl, 1093 .probe = amd_ntb_pci_probe, 1094 .remove = amd_ntb_pci_remove, 1095 }; 1096 1097 static int __init amd_ntb_pci_driver_init(void) 1098 { 1099 pr_info("%s %s\n", NTB_DESC, NTB_VER); 1100 1101 if (debugfs_initialized()) 1102 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1103 1104 return pci_register_driver(&amd_ntb_pci_driver); 1105 } 1106 module_init(amd_ntb_pci_driver_init); 1107 1108 static void __exit amd_ntb_pci_driver_exit(void) 1109 { 1110 pci_unregister_driver(&amd_ntb_pci_driver); 1111 debugfs_remove_recursive(debugfs_dir); 1112 } 1113 module_exit(amd_ntb_pci_driver_exit); 1114