1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * BSD LICENSE 14 * 15 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * * Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * * Redistributions in binary form must reproduce the above copy 24 * notice, this list of conditions and the following disclaimer in 25 * the documentation and/or other materials provided with the 26 * distribution. 27 * * Neither the name of AMD Corporation nor the names of its 28 * contributors may be used to endorse or promote products derived 29 * from this software without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * AMD PCIe NTB Linux driver 44 * 45 * Contact Information: 46 * Xiangliang Yu <Xiangliang.Yu@amd.com> 47 */ 48 49 #include <linux/debugfs.h> 50 #include <linux/delay.h> 51 #include <linux/init.h> 52 #include <linux/interrupt.h> 53 #include <linux/module.h> 54 #include <linux/acpi.h> 55 #include <linux/pci.h> 56 #include <linux/random.h> 57 #include <linux/slab.h> 58 #include <linux/ntb.h> 59 60 #include "ntb_hw_amd.h" 61 62 #define NTB_NAME "ntb_hw_amd" 63 #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" 64 #define NTB_VER "1.0" 65 66 MODULE_DESCRIPTION(NTB_DESC); 67 MODULE_VERSION(NTB_VER); 68 MODULE_LICENSE("Dual BSD/GPL"); 69 MODULE_AUTHOR("AMD Inc."); 70 71 static const struct file_operations amd_ntb_debugfs_info; 72 static struct dentry *debugfs_dir; 73 74 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) 75 { 76 if (idx < 0 || idx > ndev->mw_count) 77 return -EINVAL; 78 79 return 1 << idx; 80 } 81 82 static int amd_ntb_mw_count(struct ntb_dev *ntb) 83 { 84 return ntb_ndev(ntb)->mw_count; 85 } 86 87 static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx, 88 phys_addr_t *base, 89 resource_size_t *size, 90 resource_size_t *align, 91 resource_size_t *align_size) 92 { 93 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 94 int bar; 95 96 bar = ndev_mw_to_bar(ndev, idx); 97 if (bar < 0) 98 return bar; 99 100 if (base) 101 *base = pci_resource_start(ndev->ntb.pdev, bar); 102 103 if (size) 104 *size = pci_resource_len(ndev->ntb.pdev, bar); 105 106 if (align) 107 *align = SZ_4K; 108 109 if (align_size) 110 *align_size = 1; 111 112 return 0; 113 } 114 115 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx, 116 dma_addr_t addr, resource_size_t size) 117 { 118 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 119 unsigned long xlat_reg, limit_reg = 0; 120 resource_size_t mw_size; 121 void __iomem *mmio, *peer_mmio; 122 u64 base_addr, limit, reg_val; 123 int bar; 124 125 bar = ndev_mw_to_bar(ndev, idx); 126 if (bar < 0) 127 return bar; 128 129 mw_size = pci_resource_len(ndev->ntb.pdev, bar); 130 131 /* make sure the range fits in the usable mw size */ 132 if (size > mw_size) 133 return -EINVAL; 134 135 mmio = ndev->self_mmio; 136 peer_mmio = ndev->peer_mmio; 137 138 base_addr = pci_resource_start(ndev->ntb.pdev, bar); 139 140 if (bar != 1) { 141 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 3); 142 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 3); 143 144 /* Set the limit if supported */ 145 limit = base_addr + size; 146 147 /* set and verify setting the translation address */ 148 write64(addr, peer_mmio + xlat_reg); 149 reg_val = read64(peer_mmio + xlat_reg); 150 if (reg_val != addr) { 151 write64(0, peer_mmio + xlat_reg); 152 return -EIO; 153 } 154 155 /* set and verify setting the limit */ 156 write64(limit, mmio + limit_reg); 157 reg_val = read64(mmio + limit_reg); 158 if (reg_val != limit) { 159 write64(base_addr, mmio + limit_reg); 160 write64(0, peer_mmio + xlat_reg); 161 return -EIO; 162 } 163 } else { 164 xlat_reg = AMD_BAR1XLAT_OFFSET; 165 limit_reg = AMD_BAR1LMT_OFFSET; 166 167 /* split bar addr range must all be 32 bit */ 168 if (addr & (~0ull << 32)) 169 return -EINVAL; 170 if ((addr + size) & (~0ull << 32)) 171 return -EINVAL; 172 173 /* Set the limit if supported */ 174 limit = base_addr + size; 175 176 /* set and verify setting the translation address */ 177 write64(addr, peer_mmio + xlat_reg); 178 reg_val = read64(peer_mmio + xlat_reg); 179 if (reg_val != addr) { 180 write64(0, peer_mmio + xlat_reg); 181 return -EIO; 182 } 183 184 /* set and verify setting the limit */ 185 writel(limit, mmio + limit_reg); 186 reg_val = readl(mmio + limit_reg); 187 if (reg_val != limit) { 188 writel(base_addr, mmio + limit_reg); 189 writel(0, peer_mmio + xlat_reg); 190 return -EIO; 191 } 192 } 193 194 return 0; 195 } 196 197 static int amd_link_is_up(struct amd_ntb_dev *ndev) 198 { 199 if (!ndev->peer_sta) 200 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta); 201 202 /* If peer_sta is reset or D0 event, the ISR has 203 * started a timer to check link status of hardware. 204 * So here just clear status bit. And if peer_sta is 205 * D3 or PME_TO, D0/reset event will be happened when 206 * system wakeup/poweron, so do nothing here. 207 */ 208 if (ndev->peer_sta & AMD_PEER_RESET_EVENT) 209 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT; 210 else if (ndev->peer_sta & AMD_PEER_D0_EVENT) 211 ndev->peer_sta = 0; 212 213 return 0; 214 } 215 216 static int amd_ntb_link_is_up(struct ntb_dev *ntb, 217 enum ntb_speed *speed, 218 enum ntb_width *width) 219 { 220 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 221 int ret = 0; 222 223 if (amd_link_is_up(ndev)) { 224 if (speed) 225 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 226 if (width) 227 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 228 229 dev_dbg(ndev_dev(ndev), "link is up.\n"); 230 231 ret = 1; 232 } else { 233 if (speed) 234 *speed = NTB_SPEED_NONE; 235 if (width) 236 *width = NTB_WIDTH_NONE; 237 238 dev_dbg(ndev_dev(ndev), "link is down.\n"); 239 } 240 241 return ret; 242 } 243 244 static int amd_ntb_link_enable(struct ntb_dev *ntb, 245 enum ntb_speed max_speed, 246 enum ntb_width max_width) 247 { 248 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 249 void __iomem *mmio = ndev->self_mmio; 250 u32 ntb_ctl; 251 252 /* Enable event interrupt */ 253 ndev->int_mask &= ~AMD_EVENT_INTMASK; 254 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 255 256 if (ndev->ntb.topo == NTB_TOPO_SEC) 257 return -EINVAL; 258 dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 259 260 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 261 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); 262 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 263 264 return 0; 265 } 266 267 static int amd_ntb_link_disable(struct ntb_dev *ntb) 268 { 269 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 270 void __iomem *mmio = ndev->self_mmio; 271 u32 ntb_ctl; 272 273 /* Disable event interrupt */ 274 ndev->int_mask |= AMD_EVENT_INTMASK; 275 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 276 277 if (ndev->ntb.topo == NTB_TOPO_SEC) 278 return -EINVAL; 279 dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 280 281 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 282 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); 283 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 284 285 return 0; 286 } 287 288 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) 289 { 290 return ntb_ndev(ntb)->db_valid_mask; 291 } 292 293 static int amd_ntb_db_vector_count(struct ntb_dev *ntb) 294 { 295 return ntb_ndev(ntb)->db_count; 296 } 297 298 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 299 { 300 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 301 302 if (db_vector < 0 || db_vector > ndev->db_count) 303 return 0; 304 305 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector); 306 } 307 308 static u64 amd_ntb_db_read(struct ntb_dev *ntb) 309 { 310 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 311 void __iomem *mmio = ndev->self_mmio; 312 313 return (u64)readw(mmio + AMD_DBSTAT_OFFSET); 314 } 315 316 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 317 { 318 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 319 void __iomem *mmio = ndev->self_mmio; 320 321 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET); 322 323 return 0; 324 } 325 326 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 327 { 328 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 329 void __iomem *mmio = ndev->self_mmio; 330 unsigned long flags; 331 332 if (db_bits & ~ndev->db_valid_mask) 333 return -EINVAL; 334 335 spin_lock_irqsave(&ndev->db_mask_lock, flags); 336 ndev->db_mask |= db_bits; 337 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 338 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 339 340 return 0; 341 } 342 343 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 344 { 345 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 346 void __iomem *mmio = ndev->self_mmio; 347 unsigned long flags; 348 349 if (db_bits & ~ndev->db_valid_mask) 350 return -EINVAL; 351 352 spin_lock_irqsave(&ndev->db_mask_lock, flags); 353 ndev->db_mask &= ~db_bits; 354 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 355 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 356 357 return 0; 358 } 359 360 static int amd_ntb_peer_db_addr(struct ntb_dev *ntb, 361 phys_addr_t *db_addr, 362 resource_size_t *db_size) 363 { 364 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 365 366 if (db_addr) 367 *db_addr = (phys_addr_t)(ndev->peer_mmio + AMD_DBREQ_OFFSET); 368 if (db_size) 369 *db_size = sizeof(u32); 370 371 return 0; 372 } 373 374 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 375 { 376 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 377 void __iomem *mmio = ndev->self_mmio; 378 379 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET); 380 381 return 0; 382 } 383 384 static int amd_ntb_spad_count(struct ntb_dev *ntb) 385 { 386 return ntb_ndev(ntb)->spad_count; 387 } 388 389 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) 390 { 391 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 392 void __iomem *mmio = ndev->self_mmio; 393 u32 offset; 394 395 if (idx < 0 || idx >= ndev->spad_count) 396 return 0; 397 398 offset = ndev->self_spad + (idx << 2); 399 return readl(mmio + AMD_SPAD_OFFSET + offset); 400 } 401 402 static int amd_ntb_spad_write(struct ntb_dev *ntb, 403 int idx, u32 val) 404 { 405 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 406 void __iomem *mmio = ndev->self_mmio; 407 u32 offset; 408 409 if (idx < 0 || idx >= ndev->spad_count) 410 return -EINVAL; 411 412 offset = ndev->self_spad + (idx << 2); 413 writel(val, mmio + AMD_SPAD_OFFSET + offset); 414 415 return 0; 416 } 417 418 static int amd_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx, 419 phys_addr_t *spad_addr) 420 { 421 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 422 423 if (idx < 0 || idx >= ndev->spad_count) 424 return -EINVAL; 425 426 if (spad_addr) 427 *spad_addr = (phys_addr_t)(ndev->self_mmio + AMD_SPAD_OFFSET + 428 ndev->peer_spad + (idx << 2)); 429 return 0; 430 } 431 432 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx) 433 { 434 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 435 void __iomem *mmio = ndev->self_mmio; 436 u32 offset; 437 438 if (idx < 0 || idx >= ndev->spad_count) 439 return -EINVAL; 440 441 offset = ndev->peer_spad + (idx << 2); 442 return readl(mmio + AMD_SPAD_OFFSET + offset); 443 } 444 445 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, 446 int idx, u32 val) 447 { 448 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 449 void __iomem *mmio = ndev->self_mmio; 450 u32 offset; 451 452 if (idx < 0 || idx >= ndev->spad_count) 453 return -EINVAL; 454 455 offset = ndev->peer_spad + (idx << 2); 456 writel(val, mmio + AMD_SPAD_OFFSET + offset); 457 458 return 0; 459 } 460 461 static const struct ntb_dev_ops amd_ntb_ops = { 462 .mw_count = amd_ntb_mw_count, 463 .mw_get_range = amd_ntb_mw_get_range, 464 .mw_set_trans = amd_ntb_mw_set_trans, 465 .link_is_up = amd_ntb_link_is_up, 466 .link_enable = amd_ntb_link_enable, 467 .link_disable = amd_ntb_link_disable, 468 .db_valid_mask = amd_ntb_db_valid_mask, 469 .db_vector_count = amd_ntb_db_vector_count, 470 .db_vector_mask = amd_ntb_db_vector_mask, 471 .db_read = amd_ntb_db_read, 472 .db_clear = amd_ntb_db_clear, 473 .db_set_mask = amd_ntb_db_set_mask, 474 .db_clear_mask = amd_ntb_db_clear_mask, 475 .peer_db_addr = amd_ntb_peer_db_addr, 476 .peer_db_set = amd_ntb_peer_db_set, 477 .spad_count = amd_ntb_spad_count, 478 .spad_read = amd_ntb_spad_read, 479 .spad_write = amd_ntb_spad_write, 480 .peer_spad_addr = amd_ntb_peer_spad_addr, 481 .peer_spad_read = amd_ntb_peer_spad_read, 482 .peer_spad_write = amd_ntb_peer_spad_write, 483 }; 484 485 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) 486 { 487 void __iomem *mmio = ndev->self_mmio; 488 int reg; 489 490 reg = readl(mmio + AMD_SMUACK_OFFSET); 491 reg |= bit; 492 writel(reg, mmio + AMD_SMUACK_OFFSET); 493 494 ndev->peer_sta |= bit; 495 } 496 497 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) 498 { 499 void __iomem *mmio = ndev->self_mmio; 500 u32 status; 501 502 status = readl(mmio + AMD_INTSTAT_OFFSET); 503 if (!(status & AMD_EVENT_INTMASK)) 504 return; 505 506 dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec); 507 508 status &= AMD_EVENT_INTMASK; 509 switch (status) { 510 case AMD_PEER_FLUSH_EVENT: 511 dev_info(ndev_dev(ndev), "Flush is done.\n"); 512 break; 513 case AMD_PEER_RESET_EVENT: 514 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT); 515 516 /* link down first */ 517 ntb_link_event(&ndev->ntb); 518 /* polling peer status */ 519 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 520 521 break; 522 case AMD_PEER_D3_EVENT: 523 case AMD_PEER_PMETO_EVENT: 524 amd_ack_smu(ndev, status); 525 526 /* link down */ 527 ntb_link_event(&ndev->ntb); 528 529 break; 530 case AMD_PEER_D0_EVENT: 531 mmio = ndev->peer_mmio; 532 status = readl(mmio + AMD_PMESTAT_OFFSET); 533 /* check if this is WAKEUP event */ 534 if (status & 0x1) 535 dev_info(ndev_dev(ndev), "Wakeup is done.\n"); 536 537 amd_ack_smu(ndev, AMD_PEER_D0_EVENT); 538 539 /* start a timer to poll link status */ 540 schedule_delayed_work(&ndev->hb_timer, 541 AMD_LINK_HB_TIMEOUT); 542 break; 543 default: 544 dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status); 545 break; 546 } 547 } 548 549 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) 550 { 551 dev_dbg(ndev_dev(ndev), "vec %d\n", vec); 552 553 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) 554 amd_handle_event(ndev, vec); 555 556 if (vec < AMD_DB_CNT) 557 ntb_db_event(&ndev->ntb, vec); 558 559 return IRQ_HANDLED; 560 } 561 562 static irqreturn_t ndev_vec_isr(int irq, void *dev) 563 { 564 struct amd_ntb_vec *nvec = dev; 565 566 return ndev_interrupt(nvec->ndev, nvec->num); 567 } 568 569 static irqreturn_t ndev_irq_isr(int irq, void *dev) 570 { 571 struct amd_ntb_dev *ndev = dev; 572 573 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq); 574 } 575 576 static int ndev_init_isr(struct amd_ntb_dev *ndev, 577 int msix_min, int msix_max) 578 { 579 struct pci_dev *pdev; 580 int rc, i, msix_count, node; 581 582 pdev = ndev_pdev(ndev); 583 584 node = dev_to_node(&pdev->dev); 585 586 ndev->db_mask = ndev->db_valid_mask; 587 588 /* Try to set up msix irq */ 589 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec), 590 GFP_KERNEL, node); 591 if (!ndev->vec) 592 goto err_msix_vec_alloc; 593 594 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix), 595 GFP_KERNEL, node); 596 if (!ndev->msix) 597 goto err_msix_alloc; 598 599 for (i = 0; i < msix_max; ++i) 600 ndev->msix[i].entry = i; 601 602 msix_count = pci_enable_msix_range(pdev, ndev->msix, 603 msix_min, msix_max); 604 if (msix_count < 0) 605 goto err_msix_enable; 606 607 /* NOTE: Disable MSIX if msix count is less than 16 because of 608 * hardware limitation. 609 */ 610 if (msix_count < msix_min) { 611 pci_disable_msix(pdev); 612 goto err_msix_enable; 613 } 614 615 for (i = 0; i < msix_count; ++i) { 616 ndev->vec[i].ndev = ndev; 617 ndev->vec[i].num = i; 618 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 619 "ndev_vec_isr", &ndev->vec[i]); 620 if (rc) 621 goto err_msix_request; 622 } 623 624 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n"); 625 ndev->db_count = msix_min; 626 ndev->msix_vec_count = msix_max; 627 return 0; 628 629 err_msix_request: 630 while (i-- > 0) 631 free_irq(ndev->msix[i].vector, ndev); 632 pci_disable_msix(pdev); 633 err_msix_enable: 634 kfree(ndev->msix); 635 err_msix_alloc: 636 kfree(ndev->vec); 637 err_msix_vec_alloc: 638 ndev->msix = NULL; 639 ndev->vec = NULL; 640 641 /* Try to set up msi irq */ 642 rc = pci_enable_msi(pdev); 643 if (rc) 644 goto err_msi_enable; 645 646 rc = request_irq(pdev->irq, ndev_irq_isr, 0, 647 "ndev_irq_isr", ndev); 648 if (rc) 649 goto err_msi_request; 650 651 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n"); 652 ndev->db_count = 1; 653 ndev->msix_vec_count = 1; 654 return 0; 655 656 err_msi_request: 657 pci_disable_msi(pdev); 658 err_msi_enable: 659 660 /* Try to set up intx irq */ 661 pci_intx(pdev, 1); 662 663 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 664 "ndev_irq_isr", ndev); 665 if (rc) 666 goto err_intx_request; 667 668 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n"); 669 ndev->db_count = 1; 670 ndev->msix_vec_count = 1; 671 return 0; 672 673 err_intx_request: 674 return rc; 675 } 676 677 static void ndev_deinit_isr(struct amd_ntb_dev *ndev) 678 { 679 struct pci_dev *pdev; 680 void __iomem *mmio = ndev->self_mmio; 681 int i; 682 683 pdev = ndev_pdev(ndev); 684 685 /* Mask all doorbell interrupts */ 686 ndev->db_mask = ndev->db_valid_mask; 687 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 688 689 if (ndev->msix) { 690 i = ndev->msix_vec_count; 691 while (i--) 692 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 693 pci_disable_msix(pdev); 694 kfree(ndev->msix); 695 kfree(ndev->vec); 696 } else { 697 free_irq(pdev->irq, ndev); 698 if (pci_dev_msi_enabled(pdev)) 699 pci_disable_msi(pdev); 700 else 701 pci_intx(pdev, 0); 702 } 703 } 704 705 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 706 size_t count, loff_t *offp) 707 { 708 struct amd_ntb_dev *ndev; 709 void __iomem *mmio; 710 char *buf; 711 size_t buf_size; 712 ssize_t ret, off; 713 union { u64 v64; u32 v32; u16 v16; } u; 714 715 ndev = filp->private_data; 716 mmio = ndev->self_mmio; 717 718 buf_size = min(count, 0x800ul); 719 720 buf = kmalloc(buf_size, GFP_KERNEL); 721 if (!buf) 722 return -ENOMEM; 723 724 off = 0; 725 726 off += scnprintf(buf + off, buf_size - off, 727 "NTB Device Information:\n"); 728 729 off += scnprintf(buf + off, buf_size - off, 730 "Connection Topology -\t%s\n", 731 ntb_topo_string(ndev->ntb.topo)); 732 733 off += scnprintf(buf + off, buf_size - off, 734 "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 735 736 if (!amd_link_is_up(ndev)) { 737 off += scnprintf(buf + off, buf_size - off, 738 "Link Status -\t\tDown\n"); 739 } else { 740 off += scnprintf(buf + off, buf_size - off, 741 "Link Status -\t\tUp\n"); 742 off += scnprintf(buf + off, buf_size - off, 743 "Link Speed -\t\tPCI-E Gen %u\n", 744 NTB_LNK_STA_SPEED(ndev->lnk_sta)); 745 off += scnprintf(buf + off, buf_size - off, 746 "Link Width -\t\tx%u\n", 747 NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 748 } 749 750 off += scnprintf(buf + off, buf_size - off, 751 "Memory Window Count -\t%u\n", ndev->mw_count); 752 off += scnprintf(buf + off, buf_size - off, 753 "Scratchpad Count -\t%u\n", ndev->spad_count); 754 off += scnprintf(buf + off, buf_size - off, 755 "Doorbell Count -\t%u\n", ndev->db_count); 756 off += scnprintf(buf + off, buf_size - off, 757 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count); 758 759 off += scnprintf(buf + off, buf_size - off, 760 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 761 762 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET); 763 off += scnprintf(buf + off, buf_size - off, 764 "Doorbell Mask -\t\t\t%#06x\n", u.v32); 765 766 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET); 767 off += scnprintf(buf + off, buf_size - off, 768 "Doorbell Bell -\t\t\t%#06x\n", u.v32); 769 770 off += scnprintf(buf + off, buf_size - off, 771 "\nNTB Incoming XLAT:\n"); 772 773 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET); 774 off += scnprintf(buf + off, buf_size - off, 775 "XLAT1 -\t\t%#018llx\n", u.v64); 776 777 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET); 778 off += scnprintf(buf + off, buf_size - off, 779 "XLAT23 -\t\t%#018llx\n", u.v64); 780 781 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET); 782 off += scnprintf(buf + off, buf_size - off, 783 "XLAT45 -\t\t%#018llx\n", u.v64); 784 785 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET); 786 off += scnprintf(buf + off, buf_size - off, 787 "LMT1 -\t\t\t%#06x\n", u.v32); 788 789 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET); 790 off += scnprintf(buf + off, buf_size - off, 791 "LMT23 -\t\t\t%#018llx\n", u.v64); 792 793 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET); 794 off += scnprintf(buf + off, buf_size - off, 795 "LMT45 -\t\t\t%#018llx\n", u.v64); 796 797 ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 798 kfree(buf); 799 return ret; 800 } 801 802 static void ndev_init_debugfs(struct amd_ntb_dev *ndev) 803 { 804 if (!debugfs_dir) { 805 ndev->debugfs_dir = NULL; 806 ndev->debugfs_info = NULL; 807 } else { 808 ndev->debugfs_dir = 809 debugfs_create_dir(ndev_name(ndev), debugfs_dir); 810 if (!ndev->debugfs_dir) 811 ndev->debugfs_info = NULL; 812 else 813 ndev->debugfs_info = 814 debugfs_create_file("info", S_IRUSR, 815 ndev->debugfs_dir, ndev, 816 &amd_ntb_debugfs_info); 817 } 818 } 819 820 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) 821 { 822 debugfs_remove_recursive(ndev->debugfs_dir); 823 } 824 825 static inline void ndev_init_struct(struct amd_ntb_dev *ndev, 826 struct pci_dev *pdev) 827 { 828 ndev->ntb.pdev = pdev; 829 ndev->ntb.topo = NTB_TOPO_NONE; 830 ndev->ntb.ops = &amd_ntb_ops; 831 ndev->int_mask = AMD_EVENT_INTMASK; 832 spin_lock_init(&ndev->db_mask_lock); 833 } 834 835 static int amd_poll_link(struct amd_ntb_dev *ndev) 836 { 837 void __iomem *mmio = ndev->peer_mmio; 838 u32 reg, stat; 839 int rc; 840 841 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 842 reg &= NTB_LIN_STA_ACTIVE_BIT; 843 844 dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg); 845 846 if (reg == ndev->cntl_sta) 847 return 0; 848 849 ndev->cntl_sta = reg; 850 851 rc = pci_read_config_dword(ndev->ntb.pdev, 852 AMD_LINK_STATUS_OFFSET, &stat); 853 if (rc) 854 return 0; 855 ndev->lnk_sta = stat; 856 857 return 1; 858 } 859 860 static void amd_link_hb(struct work_struct *work) 861 { 862 struct amd_ntb_dev *ndev = hb_ndev(work); 863 864 if (amd_poll_link(ndev)) 865 ntb_link_event(&ndev->ntb); 866 867 if (!amd_link_is_up(ndev)) 868 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 869 } 870 871 static int amd_init_isr(struct amd_ntb_dev *ndev) 872 { 873 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); 874 } 875 876 static void amd_init_side_info(struct amd_ntb_dev *ndev) 877 { 878 void __iomem *mmio = ndev->self_mmio; 879 unsigned int reg; 880 881 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 882 if (!(reg & AMD_SIDE_READY)) { 883 reg |= AMD_SIDE_READY; 884 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 885 } 886 } 887 888 static void amd_deinit_side_info(struct amd_ntb_dev *ndev) 889 { 890 void __iomem *mmio = ndev->self_mmio; 891 unsigned int reg; 892 893 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 894 if (reg & AMD_SIDE_READY) { 895 reg &= ~AMD_SIDE_READY; 896 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 897 readl(mmio + AMD_SIDEINFO_OFFSET); 898 } 899 } 900 901 static int amd_init_ntb(struct amd_ntb_dev *ndev) 902 { 903 void __iomem *mmio = ndev->self_mmio; 904 905 ndev->mw_count = AMD_MW_CNT; 906 ndev->spad_count = AMD_SPADS_CNT; 907 ndev->db_count = AMD_DB_CNT; 908 909 switch (ndev->ntb.topo) { 910 case NTB_TOPO_PRI: 911 case NTB_TOPO_SEC: 912 ndev->spad_count >>= 1; 913 if (ndev->ntb.topo == NTB_TOPO_PRI) { 914 ndev->self_spad = 0; 915 ndev->peer_spad = 0x20; 916 } else { 917 ndev->self_spad = 0x20; 918 ndev->peer_spad = 0; 919 } 920 921 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); 922 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 923 924 break; 925 default: 926 dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n"); 927 return -EINVAL; 928 } 929 930 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 931 932 /* Mask event interrupts */ 933 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 934 935 return 0; 936 } 937 938 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) 939 { 940 void __iomem *mmio = ndev->self_mmio; 941 u32 info; 942 943 info = readl(mmio + AMD_SIDEINFO_OFFSET); 944 if (info & AMD_SIDE_MASK) 945 return NTB_TOPO_SEC; 946 else 947 return NTB_TOPO_PRI; 948 } 949 950 static int amd_init_dev(struct amd_ntb_dev *ndev) 951 { 952 struct pci_dev *pdev; 953 int rc = 0; 954 955 pdev = ndev_pdev(ndev); 956 957 ndev->ntb.topo = amd_get_topo(ndev); 958 dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n", 959 ntb_topo_string(ndev->ntb.topo)); 960 961 rc = amd_init_ntb(ndev); 962 if (rc) 963 return rc; 964 965 rc = amd_init_isr(ndev); 966 if (rc) { 967 dev_err(ndev_dev(ndev), "fail to init isr.\n"); 968 return rc; 969 } 970 971 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 972 973 return 0; 974 } 975 976 static void amd_deinit_dev(struct amd_ntb_dev *ndev) 977 { 978 cancel_delayed_work_sync(&ndev->hb_timer); 979 980 ndev_deinit_isr(ndev); 981 } 982 983 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, 984 struct pci_dev *pdev) 985 { 986 int rc; 987 988 pci_set_drvdata(pdev, ndev); 989 990 rc = pci_enable_device(pdev); 991 if (rc) 992 goto err_pci_enable; 993 994 rc = pci_request_regions(pdev, NTB_NAME); 995 if (rc) 996 goto err_pci_regions; 997 998 pci_set_master(pdev); 999 1000 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1001 if (rc) { 1002 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 1003 if (rc) 1004 goto err_dma_mask; 1005 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n"); 1006 } 1007 1008 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1009 if (rc) { 1010 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 1011 if (rc) 1012 goto err_dma_mask; 1013 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n"); 1014 } 1015 1016 ndev->self_mmio = pci_iomap(pdev, 0, 0); 1017 if (!ndev->self_mmio) { 1018 rc = -EIO; 1019 goto err_dma_mask; 1020 } 1021 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; 1022 1023 return 0; 1024 1025 err_dma_mask: 1026 pci_clear_master(pdev); 1027 err_pci_regions: 1028 pci_disable_device(pdev); 1029 err_pci_enable: 1030 pci_set_drvdata(pdev, NULL); 1031 return rc; 1032 } 1033 1034 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) 1035 { 1036 struct pci_dev *pdev = ndev_pdev(ndev); 1037 1038 pci_iounmap(pdev, ndev->self_mmio); 1039 1040 pci_clear_master(pdev); 1041 pci_release_regions(pdev); 1042 pci_disable_device(pdev); 1043 pci_set_drvdata(pdev, NULL); 1044 } 1045 1046 static int amd_ntb_pci_probe(struct pci_dev *pdev, 1047 const struct pci_device_id *id) 1048 { 1049 struct amd_ntb_dev *ndev; 1050 int rc, node; 1051 1052 node = dev_to_node(&pdev->dev); 1053 1054 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1055 if (!ndev) { 1056 rc = -ENOMEM; 1057 goto err_ndev; 1058 } 1059 1060 ndev_init_struct(ndev, pdev); 1061 1062 rc = amd_ntb_init_pci(ndev, pdev); 1063 if (rc) 1064 goto err_init_pci; 1065 1066 rc = amd_init_dev(ndev); 1067 if (rc) 1068 goto err_init_dev; 1069 1070 /* write side info */ 1071 amd_init_side_info(ndev); 1072 1073 amd_poll_link(ndev); 1074 1075 ndev_init_debugfs(ndev); 1076 1077 rc = ntb_register_device(&ndev->ntb); 1078 if (rc) 1079 goto err_register; 1080 1081 dev_info(&pdev->dev, "NTB device registered.\n"); 1082 1083 return 0; 1084 1085 err_register: 1086 ndev_deinit_debugfs(ndev); 1087 amd_deinit_dev(ndev); 1088 err_init_dev: 1089 amd_ntb_deinit_pci(ndev); 1090 err_init_pci: 1091 kfree(ndev); 1092 err_ndev: 1093 return rc; 1094 } 1095 1096 static void amd_ntb_pci_remove(struct pci_dev *pdev) 1097 { 1098 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); 1099 1100 ntb_unregister_device(&ndev->ntb); 1101 ndev_deinit_debugfs(ndev); 1102 amd_deinit_side_info(ndev); 1103 amd_deinit_dev(ndev); 1104 amd_ntb_deinit_pci(ndev); 1105 kfree(ndev); 1106 } 1107 1108 static const struct file_operations amd_ntb_debugfs_info = { 1109 .owner = THIS_MODULE, 1110 .open = simple_open, 1111 .read = ndev_debugfs_read, 1112 }; 1113 1114 static const struct pci_device_id amd_ntb_pci_tbl[] = { 1115 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)}, 1116 {0} 1117 }; 1118 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); 1119 1120 static struct pci_driver amd_ntb_pci_driver = { 1121 .name = KBUILD_MODNAME, 1122 .id_table = amd_ntb_pci_tbl, 1123 .probe = amd_ntb_pci_probe, 1124 .remove = amd_ntb_pci_remove, 1125 }; 1126 1127 static int __init amd_ntb_pci_driver_init(void) 1128 { 1129 pr_info("%s %s\n", NTB_DESC, NTB_VER); 1130 1131 if (debugfs_initialized()) 1132 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1133 1134 return pci_register_driver(&amd_ntb_pci_driver); 1135 } 1136 module_init(amd_ntb_pci_driver_init); 1137 1138 static void __exit amd_ntb_pci_driver_exit(void) 1139 { 1140 pci_unregister_driver(&amd_ntb_pci_driver); 1141 debugfs_remove_recursive(debugfs_dir); 1142 } 1143 module_exit(amd_ntb_pci_driver_exit); 1144