1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 8 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 17 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of AMD Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * AMD PCIe NTB Linux driver 46 * 47 * Contact Information: 48 * Xiangliang Yu <Xiangliang.Yu@amd.com> 49 */ 50 51 #include <linux/debugfs.h> 52 #include <linux/delay.h> 53 #include <linux/init.h> 54 #include <linux/interrupt.h> 55 #include <linux/module.h> 56 #include <linux/acpi.h> 57 #include <linux/pci.h> 58 #include <linux/random.h> 59 #include <linux/slab.h> 60 #include <linux/ntb.h> 61 62 #include "ntb_hw_amd.h" 63 64 #define NTB_NAME "ntb_hw_amd" 65 #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" 66 #define NTB_VER "1.0" 67 68 MODULE_DESCRIPTION(NTB_DESC); 69 MODULE_VERSION(NTB_VER); 70 MODULE_LICENSE("Dual BSD/GPL"); 71 MODULE_AUTHOR("AMD Inc."); 72 73 static const struct file_operations amd_ntb_debugfs_info; 74 static struct dentry *debugfs_dir; 75 76 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) 77 { 78 if (idx < 0 || idx > ndev->mw_count) 79 return -EINVAL; 80 81 return ndev->dev_data->mw_idx << idx; 82 } 83 84 static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx) 85 { 86 if (pidx != NTB_DEF_PEER_IDX) 87 return -EINVAL; 88 89 return ntb_ndev(ntb)->mw_count; 90 } 91 92 static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, 93 resource_size_t *addr_align, 94 resource_size_t *size_align, 95 resource_size_t *size_max) 96 { 97 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 98 int bar; 99 100 if (pidx != NTB_DEF_PEER_IDX) 101 return -EINVAL; 102 103 bar = ndev_mw_to_bar(ndev, idx); 104 if (bar < 0) 105 return bar; 106 107 if (addr_align) 108 *addr_align = SZ_4K; 109 110 if (size_align) 111 *size_align = 1; 112 113 if (size_max) 114 *size_max = pci_resource_len(ndev->ntb.pdev, bar); 115 116 return 0; 117 } 118 119 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, 120 dma_addr_t addr, resource_size_t size) 121 { 122 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 123 unsigned long xlat_reg, limit_reg = 0; 124 resource_size_t mw_size; 125 void __iomem *mmio, *peer_mmio; 126 u64 base_addr, limit, reg_val; 127 int bar; 128 129 if (pidx != NTB_DEF_PEER_IDX) 130 return -EINVAL; 131 132 bar = ndev_mw_to_bar(ndev, idx); 133 if (bar < 0) 134 return bar; 135 136 mw_size = pci_resource_len(ntb->pdev, bar); 137 138 /* make sure the range fits in the usable mw size */ 139 if (size > mw_size) 140 return -EINVAL; 141 142 mmio = ndev->self_mmio; 143 peer_mmio = ndev->peer_mmio; 144 145 base_addr = pci_resource_start(ntb->pdev, bar); 146 147 if (bar != 1) { 148 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2); 149 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2); 150 151 /* Set the limit if supported */ 152 limit = size; 153 154 /* set and verify setting the translation address */ 155 write64(addr, peer_mmio + xlat_reg); 156 reg_val = read64(peer_mmio + xlat_reg); 157 if (reg_val != addr) { 158 write64(0, peer_mmio + xlat_reg); 159 return -EIO; 160 } 161 162 /* set and verify setting the limit */ 163 write64(limit, peer_mmio + limit_reg); 164 reg_val = read64(peer_mmio + limit_reg); 165 if (reg_val != limit) { 166 write64(base_addr, mmio + limit_reg); 167 write64(0, peer_mmio + xlat_reg); 168 return -EIO; 169 } 170 } else { 171 xlat_reg = AMD_BAR1XLAT_OFFSET; 172 limit_reg = AMD_BAR1LMT_OFFSET; 173 174 /* Set the limit if supported */ 175 limit = size; 176 177 /* set and verify setting the translation address */ 178 write64(addr, peer_mmio + xlat_reg); 179 reg_val = read64(peer_mmio + xlat_reg); 180 if (reg_val != addr) { 181 write64(0, peer_mmio + xlat_reg); 182 return -EIO; 183 } 184 185 /* set and verify setting the limit */ 186 writel(limit, peer_mmio + limit_reg); 187 reg_val = readl(peer_mmio + limit_reg); 188 if (reg_val != limit) { 189 writel(base_addr, mmio + limit_reg); 190 writel(0, peer_mmio + xlat_reg); 191 return -EIO; 192 } 193 } 194 195 return 0; 196 } 197 198 static int amd_link_is_up(struct amd_ntb_dev *ndev) 199 { 200 if (!ndev->peer_sta) 201 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta); 202 203 if (ndev->peer_sta & AMD_LINK_UP_EVENT) { 204 ndev->peer_sta = 0; 205 return 1; 206 } 207 208 /* If peer_sta is reset or D0 event, the ISR has 209 * started a timer to check link status of hardware. 210 * So here just clear status bit. And if peer_sta is 211 * D3 or PME_TO, D0/reset event will be happened when 212 * system wakeup/poweron, so do nothing here. 213 */ 214 if (ndev->peer_sta & AMD_PEER_RESET_EVENT) 215 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT; 216 else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT)) 217 ndev->peer_sta = 0; 218 219 return 0; 220 } 221 222 static u64 amd_ntb_link_is_up(struct ntb_dev *ntb, 223 enum ntb_speed *speed, 224 enum ntb_width *width) 225 { 226 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 227 int ret = 0; 228 229 if (amd_link_is_up(ndev)) { 230 if (speed) 231 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 232 if (width) 233 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 234 235 dev_dbg(&ntb->pdev->dev, "link is up.\n"); 236 237 ret = 1; 238 } else { 239 if (speed) 240 *speed = NTB_SPEED_NONE; 241 if (width) 242 *width = NTB_WIDTH_NONE; 243 244 dev_dbg(&ntb->pdev->dev, "link is down.\n"); 245 } 246 247 return ret; 248 } 249 250 static int amd_ntb_link_enable(struct ntb_dev *ntb, 251 enum ntb_speed max_speed, 252 enum ntb_width max_width) 253 { 254 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 255 void __iomem *mmio = ndev->self_mmio; 256 u32 ntb_ctl; 257 258 /* Enable event interrupt */ 259 ndev->int_mask &= ~AMD_EVENT_INTMASK; 260 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 261 262 if (ndev->ntb.topo == NTB_TOPO_SEC) 263 return -EINVAL; 264 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); 265 266 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 267 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); 268 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 269 270 return 0; 271 } 272 273 static int amd_ntb_link_disable(struct ntb_dev *ntb) 274 { 275 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 276 void __iomem *mmio = ndev->self_mmio; 277 u32 ntb_ctl; 278 279 /* Disable event interrupt */ 280 ndev->int_mask |= AMD_EVENT_INTMASK; 281 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 282 283 if (ndev->ntb.topo == NTB_TOPO_SEC) 284 return -EINVAL; 285 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); 286 287 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 288 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); 289 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 290 291 return 0; 292 } 293 294 static int amd_ntb_peer_mw_count(struct ntb_dev *ntb) 295 { 296 /* The same as for inbound MWs */ 297 return ntb_ndev(ntb)->mw_count; 298 } 299 300 static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 301 phys_addr_t *base, resource_size_t *size) 302 { 303 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 304 int bar; 305 306 bar = ndev_mw_to_bar(ndev, idx); 307 if (bar < 0) 308 return bar; 309 310 if (base) 311 *base = pci_resource_start(ndev->ntb.pdev, bar); 312 313 if (size) 314 *size = pci_resource_len(ndev->ntb.pdev, bar); 315 316 return 0; 317 } 318 319 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) 320 { 321 return ntb_ndev(ntb)->db_valid_mask; 322 } 323 324 static int amd_ntb_db_vector_count(struct ntb_dev *ntb) 325 { 326 return ntb_ndev(ntb)->db_count; 327 } 328 329 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 330 { 331 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 332 333 if (db_vector < 0 || db_vector > ndev->db_count) 334 return 0; 335 336 return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector); 337 } 338 339 static u64 amd_ntb_db_read(struct ntb_dev *ntb) 340 { 341 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 342 void __iomem *mmio = ndev->self_mmio; 343 344 return (u64)readw(mmio + AMD_DBSTAT_OFFSET); 345 } 346 347 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 348 { 349 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 350 void __iomem *mmio = ndev->self_mmio; 351 352 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET); 353 354 return 0; 355 } 356 357 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 358 { 359 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 360 void __iomem *mmio = ndev->self_mmio; 361 unsigned long flags; 362 363 if (db_bits & ~ndev->db_valid_mask) 364 return -EINVAL; 365 366 spin_lock_irqsave(&ndev->db_mask_lock, flags); 367 ndev->db_mask |= db_bits; 368 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 369 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 370 371 return 0; 372 } 373 374 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 375 { 376 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 377 void __iomem *mmio = ndev->self_mmio; 378 unsigned long flags; 379 380 if (db_bits & ~ndev->db_valid_mask) 381 return -EINVAL; 382 383 spin_lock_irqsave(&ndev->db_mask_lock, flags); 384 ndev->db_mask &= ~db_bits; 385 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 386 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 387 388 return 0; 389 } 390 391 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 392 { 393 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 394 void __iomem *mmio = ndev->self_mmio; 395 396 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET); 397 398 return 0; 399 } 400 401 static int amd_ntb_spad_count(struct ntb_dev *ntb) 402 { 403 return ntb_ndev(ntb)->spad_count; 404 } 405 406 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) 407 { 408 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 409 void __iomem *mmio = ndev->self_mmio; 410 u32 offset; 411 412 if (idx < 0 || idx >= ndev->spad_count) 413 return 0; 414 415 offset = ndev->self_spad + (idx << 2); 416 return readl(mmio + AMD_SPAD_OFFSET + offset); 417 } 418 419 static int amd_ntb_spad_write(struct ntb_dev *ntb, 420 int idx, u32 val) 421 { 422 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 423 void __iomem *mmio = ndev->self_mmio; 424 u32 offset; 425 426 if (idx < 0 || idx >= ndev->spad_count) 427 return -EINVAL; 428 429 offset = ndev->self_spad + (idx << 2); 430 writel(val, mmio + AMD_SPAD_OFFSET + offset); 431 432 return 0; 433 } 434 435 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) 436 { 437 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 438 void __iomem *mmio = ndev->self_mmio; 439 u32 offset; 440 441 if (sidx < 0 || sidx >= ndev->spad_count) 442 return -EINVAL; 443 444 offset = ndev->peer_spad + (sidx << 2); 445 return readl(mmio + AMD_SPAD_OFFSET + offset); 446 } 447 448 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, 449 int sidx, u32 val) 450 { 451 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 452 void __iomem *mmio = ndev->self_mmio; 453 u32 offset; 454 455 if (sidx < 0 || sidx >= ndev->spad_count) 456 return -EINVAL; 457 458 offset = ndev->peer_spad + (sidx << 2); 459 writel(val, mmio + AMD_SPAD_OFFSET + offset); 460 461 return 0; 462 } 463 464 static const struct ntb_dev_ops amd_ntb_ops = { 465 .mw_count = amd_ntb_mw_count, 466 .mw_get_align = amd_ntb_mw_get_align, 467 .mw_set_trans = amd_ntb_mw_set_trans, 468 .peer_mw_count = amd_ntb_peer_mw_count, 469 .peer_mw_get_addr = amd_ntb_peer_mw_get_addr, 470 .link_is_up = amd_ntb_link_is_up, 471 .link_enable = amd_ntb_link_enable, 472 .link_disable = amd_ntb_link_disable, 473 .db_valid_mask = amd_ntb_db_valid_mask, 474 .db_vector_count = amd_ntb_db_vector_count, 475 .db_vector_mask = amd_ntb_db_vector_mask, 476 .db_read = amd_ntb_db_read, 477 .db_clear = amd_ntb_db_clear, 478 .db_set_mask = amd_ntb_db_set_mask, 479 .db_clear_mask = amd_ntb_db_clear_mask, 480 .peer_db_set = amd_ntb_peer_db_set, 481 .spad_count = amd_ntb_spad_count, 482 .spad_read = amd_ntb_spad_read, 483 .spad_write = amd_ntb_spad_write, 484 .peer_spad_read = amd_ntb_peer_spad_read, 485 .peer_spad_write = amd_ntb_peer_spad_write, 486 }; 487 488 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) 489 { 490 void __iomem *mmio = ndev->self_mmio; 491 int reg; 492 493 reg = readl(mmio + AMD_SMUACK_OFFSET); 494 reg |= bit; 495 writel(reg, mmio + AMD_SMUACK_OFFSET); 496 497 ndev->peer_sta |= bit; 498 } 499 500 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) 501 { 502 void __iomem *mmio = ndev->self_mmio; 503 struct device *dev = &ndev->ntb.pdev->dev; 504 u32 status; 505 506 status = readl(mmio + AMD_INTSTAT_OFFSET); 507 if (!(status & AMD_EVENT_INTMASK)) 508 return; 509 510 dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec); 511 512 status &= AMD_EVENT_INTMASK; 513 switch (status) { 514 case AMD_PEER_FLUSH_EVENT: 515 dev_info(dev, "Flush is done.\n"); 516 break; 517 case AMD_PEER_RESET_EVENT: 518 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT); 519 520 /* link down first */ 521 ntb_link_event(&ndev->ntb); 522 /* polling peer status */ 523 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 524 525 break; 526 case AMD_PEER_D3_EVENT: 527 case AMD_PEER_PMETO_EVENT: 528 case AMD_LINK_UP_EVENT: 529 case AMD_LINK_DOWN_EVENT: 530 amd_ack_smu(ndev, status); 531 532 /* link down */ 533 ntb_link_event(&ndev->ntb); 534 535 break; 536 case AMD_PEER_D0_EVENT: 537 mmio = ndev->peer_mmio; 538 status = readl(mmio + AMD_PMESTAT_OFFSET); 539 /* check if this is WAKEUP event */ 540 if (status & 0x1) 541 dev_info(dev, "Wakeup is done.\n"); 542 543 amd_ack_smu(ndev, AMD_PEER_D0_EVENT); 544 545 /* start a timer to poll link status */ 546 schedule_delayed_work(&ndev->hb_timer, 547 AMD_LINK_HB_TIMEOUT); 548 break; 549 default: 550 dev_info(dev, "event status = 0x%x.\n", status); 551 break; 552 } 553 } 554 555 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) 556 { 557 dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec); 558 559 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) 560 amd_handle_event(ndev, vec); 561 562 if (vec < AMD_DB_CNT) 563 ntb_db_event(&ndev->ntb, vec); 564 565 return IRQ_HANDLED; 566 } 567 568 static irqreturn_t ndev_vec_isr(int irq, void *dev) 569 { 570 struct amd_ntb_vec *nvec = dev; 571 572 return ndev_interrupt(nvec->ndev, nvec->num); 573 } 574 575 static irqreturn_t ndev_irq_isr(int irq, void *dev) 576 { 577 struct amd_ntb_dev *ndev = dev; 578 579 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq); 580 } 581 582 static int ndev_init_isr(struct amd_ntb_dev *ndev, 583 int msix_min, int msix_max) 584 { 585 struct pci_dev *pdev; 586 int rc, i, msix_count, node; 587 588 pdev = ndev->ntb.pdev; 589 590 node = dev_to_node(&pdev->dev); 591 592 ndev->db_mask = ndev->db_valid_mask; 593 594 /* Try to set up msix irq */ 595 ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec), 596 GFP_KERNEL, node); 597 if (!ndev->vec) 598 goto err_msix_vec_alloc; 599 600 ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix), 601 GFP_KERNEL, node); 602 if (!ndev->msix) 603 goto err_msix_alloc; 604 605 for (i = 0; i < msix_max; ++i) 606 ndev->msix[i].entry = i; 607 608 msix_count = pci_enable_msix_range(pdev, ndev->msix, 609 msix_min, msix_max); 610 if (msix_count < 0) 611 goto err_msix_enable; 612 613 /* NOTE: Disable MSIX if msix count is less than 16 because of 614 * hardware limitation. 615 */ 616 if (msix_count < msix_min) { 617 pci_disable_msix(pdev); 618 goto err_msix_enable; 619 } 620 621 for (i = 0; i < msix_count; ++i) { 622 ndev->vec[i].ndev = ndev; 623 ndev->vec[i].num = i; 624 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 625 "ndev_vec_isr", &ndev->vec[i]); 626 if (rc) 627 goto err_msix_request; 628 } 629 630 dev_dbg(&pdev->dev, "Using msix interrupts\n"); 631 ndev->db_count = msix_min; 632 ndev->msix_vec_count = msix_max; 633 return 0; 634 635 err_msix_request: 636 while (i-- > 0) 637 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 638 pci_disable_msix(pdev); 639 err_msix_enable: 640 kfree(ndev->msix); 641 err_msix_alloc: 642 kfree(ndev->vec); 643 err_msix_vec_alloc: 644 ndev->msix = NULL; 645 ndev->vec = NULL; 646 647 /* Try to set up msi irq */ 648 rc = pci_enable_msi(pdev); 649 if (rc) 650 goto err_msi_enable; 651 652 rc = request_irq(pdev->irq, ndev_irq_isr, 0, 653 "ndev_irq_isr", ndev); 654 if (rc) 655 goto err_msi_request; 656 657 dev_dbg(&pdev->dev, "Using msi interrupts\n"); 658 ndev->db_count = 1; 659 ndev->msix_vec_count = 1; 660 return 0; 661 662 err_msi_request: 663 pci_disable_msi(pdev); 664 err_msi_enable: 665 666 /* Try to set up intx irq */ 667 pci_intx(pdev, 1); 668 669 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 670 "ndev_irq_isr", ndev); 671 if (rc) 672 goto err_intx_request; 673 674 dev_dbg(&pdev->dev, "Using intx interrupts\n"); 675 ndev->db_count = 1; 676 ndev->msix_vec_count = 1; 677 return 0; 678 679 err_intx_request: 680 return rc; 681 } 682 683 static void ndev_deinit_isr(struct amd_ntb_dev *ndev) 684 { 685 struct pci_dev *pdev; 686 void __iomem *mmio = ndev->self_mmio; 687 int i; 688 689 pdev = ndev->ntb.pdev; 690 691 /* Mask all doorbell interrupts */ 692 ndev->db_mask = ndev->db_valid_mask; 693 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 694 695 if (ndev->msix) { 696 i = ndev->msix_vec_count; 697 while (i--) 698 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 699 pci_disable_msix(pdev); 700 kfree(ndev->msix); 701 kfree(ndev->vec); 702 } else { 703 free_irq(pdev->irq, ndev); 704 if (pci_dev_msi_enabled(pdev)) 705 pci_disable_msi(pdev); 706 else 707 pci_intx(pdev, 0); 708 } 709 } 710 711 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 712 size_t count, loff_t *offp) 713 { 714 struct amd_ntb_dev *ndev; 715 void __iomem *mmio; 716 char *buf; 717 size_t buf_size; 718 ssize_t ret, off; 719 union { u64 v64; u32 v32; u16 v16; } u; 720 721 ndev = filp->private_data; 722 mmio = ndev->self_mmio; 723 724 buf_size = min(count, 0x800ul); 725 726 buf = kmalloc(buf_size, GFP_KERNEL); 727 if (!buf) 728 return -ENOMEM; 729 730 off = 0; 731 732 off += scnprintf(buf + off, buf_size - off, 733 "NTB Device Information:\n"); 734 735 off += scnprintf(buf + off, buf_size - off, 736 "Connection Topology -\t%s\n", 737 ntb_topo_string(ndev->ntb.topo)); 738 739 off += scnprintf(buf + off, buf_size - off, 740 "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 741 742 if (!amd_link_is_up(ndev)) { 743 off += scnprintf(buf + off, buf_size - off, 744 "Link Status -\t\tDown\n"); 745 } else { 746 off += scnprintf(buf + off, buf_size - off, 747 "Link Status -\t\tUp\n"); 748 off += scnprintf(buf + off, buf_size - off, 749 "Link Speed -\t\tPCI-E Gen %u\n", 750 NTB_LNK_STA_SPEED(ndev->lnk_sta)); 751 off += scnprintf(buf + off, buf_size - off, 752 "Link Width -\t\tx%u\n", 753 NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 754 } 755 756 off += scnprintf(buf + off, buf_size - off, 757 "Memory Window Count -\t%u\n", ndev->mw_count); 758 off += scnprintf(buf + off, buf_size - off, 759 "Scratchpad Count -\t%u\n", ndev->spad_count); 760 off += scnprintf(buf + off, buf_size - off, 761 "Doorbell Count -\t%u\n", ndev->db_count); 762 off += scnprintf(buf + off, buf_size - off, 763 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count); 764 765 off += scnprintf(buf + off, buf_size - off, 766 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 767 768 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET); 769 off += scnprintf(buf + off, buf_size - off, 770 "Doorbell Mask -\t\t\t%#06x\n", u.v32); 771 772 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET); 773 off += scnprintf(buf + off, buf_size - off, 774 "Doorbell Bell -\t\t\t%#06x\n", u.v32); 775 776 off += scnprintf(buf + off, buf_size - off, 777 "\nNTB Incoming XLAT:\n"); 778 779 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET); 780 off += scnprintf(buf + off, buf_size - off, 781 "XLAT1 -\t\t%#018llx\n", u.v64); 782 783 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET); 784 off += scnprintf(buf + off, buf_size - off, 785 "XLAT23 -\t\t%#018llx\n", u.v64); 786 787 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET); 788 off += scnprintf(buf + off, buf_size - off, 789 "XLAT45 -\t\t%#018llx\n", u.v64); 790 791 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET); 792 off += scnprintf(buf + off, buf_size - off, 793 "LMT1 -\t\t\t%#06x\n", u.v32); 794 795 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET); 796 off += scnprintf(buf + off, buf_size - off, 797 "LMT23 -\t\t\t%#018llx\n", u.v64); 798 799 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET); 800 off += scnprintf(buf + off, buf_size - off, 801 "LMT45 -\t\t\t%#018llx\n", u.v64); 802 803 ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 804 kfree(buf); 805 return ret; 806 } 807 808 static void ndev_init_debugfs(struct amd_ntb_dev *ndev) 809 { 810 if (!debugfs_dir) { 811 ndev->debugfs_dir = NULL; 812 ndev->debugfs_info = NULL; 813 } else { 814 ndev->debugfs_dir = 815 debugfs_create_dir(pci_name(ndev->ntb.pdev), 816 debugfs_dir); 817 if (!ndev->debugfs_dir) 818 ndev->debugfs_info = NULL; 819 else 820 ndev->debugfs_info = 821 debugfs_create_file("info", S_IRUSR, 822 ndev->debugfs_dir, ndev, 823 &amd_ntb_debugfs_info); 824 } 825 } 826 827 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) 828 { 829 debugfs_remove_recursive(ndev->debugfs_dir); 830 } 831 832 static inline void ndev_init_struct(struct amd_ntb_dev *ndev, 833 struct pci_dev *pdev) 834 { 835 ndev->ntb.pdev = pdev; 836 ndev->ntb.topo = NTB_TOPO_NONE; 837 ndev->ntb.ops = &amd_ntb_ops; 838 ndev->int_mask = AMD_EVENT_INTMASK; 839 spin_lock_init(&ndev->db_mask_lock); 840 } 841 842 static int amd_poll_link(struct amd_ntb_dev *ndev) 843 { 844 void __iomem *mmio = ndev->peer_mmio; 845 u32 reg, stat; 846 int rc; 847 848 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 849 reg &= NTB_LIN_STA_ACTIVE_BIT; 850 851 dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg); 852 853 if (reg == ndev->cntl_sta) 854 return 0; 855 856 ndev->cntl_sta = reg; 857 858 rc = pci_read_config_dword(ndev->ntb.pdev, 859 AMD_LINK_STATUS_OFFSET, &stat); 860 if (rc) 861 return 0; 862 ndev->lnk_sta = stat; 863 864 return 1; 865 } 866 867 static void amd_link_hb(struct work_struct *work) 868 { 869 struct amd_ntb_dev *ndev = hb_ndev(work); 870 871 if (amd_poll_link(ndev)) 872 ntb_link_event(&ndev->ntb); 873 874 if (!amd_link_is_up(ndev)) 875 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 876 } 877 878 static int amd_init_isr(struct amd_ntb_dev *ndev) 879 { 880 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); 881 } 882 883 static void amd_init_side_info(struct amd_ntb_dev *ndev) 884 { 885 void __iomem *mmio = ndev->self_mmio; 886 unsigned int reg; 887 888 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 889 if (!(reg & AMD_SIDE_READY)) { 890 reg |= AMD_SIDE_READY; 891 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 892 } 893 } 894 895 static void amd_deinit_side_info(struct amd_ntb_dev *ndev) 896 { 897 void __iomem *mmio = ndev->self_mmio; 898 unsigned int reg; 899 900 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 901 if (reg & AMD_SIDE_READY) { 902 reg &= ~AMD_SIDE_READY; 903 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 904 readl(mmio + AMD_SIDEINFO_OFFSET); 905 } 906 } 907 908 static int amd_init_ntb(struct amd_ntb_dev *ndev) 909 { 910 void __iomem *mmio = ndev->self_mmio; 911 912 ndev->mw_count = ndev->dev_data->mw_count; 913 ndev->spad_count = AMD_SPADS_CNT; 914 ndev->db_count = AMD_DB_CNT; 915 916 switch (ndev->ntb.topo) { 917 case NTB_TOPO_PRI: 918 case NTB_TOPO_SEC: 919 ndev->spad_count >>= 1; 920 if (ndev->ntb.topo == NTB_TOPO_PRI) { 921 ndev->self_spad = 0; 922 ndev->peer_spad = 0x20; 923 } else { 924 ndev->self_spad = 0x20; 925 ndev->peer_spad = 0; 926 } 927 928 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); 929 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 930 931 break; 932 default: 933 dev_err(&ndev->ntb.pdev->dev, 934 "AMD NTB does not support B2B mode.\n"); 935 return -EINVAL; 936 } 937 938 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 939 940 /* Mask event interrupts */ 941 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 942 943 return 0; 944 } 945 946 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) 947 { 948 void __iomem *mmio = ndev->self_mmio; 949 u32 info; 950 951 info = readl(mmio + AMD_SIDEINFO_OFFSET); 952 if (info & AMD_SIDE_MASK) 953 return NTB_TOPO_SEC; 954 else 955 return NTB_TOPO_PRI; 956 } 957 958 static int amd_init_dev(struct amd_ntb_dev *ndev) 959 { 960 struct pci_dev *pdev; 961 int rc = 0; 962 963 pdev = ndev->ntb.pdev; 964 965 ndev->ntb.topo = amd_get_topo(ndev); 966 dev_dbg(&pdev->dev, "AMD NTB topo is %s\n", 967 ntb_topo_string(ndev->ntb.topo)); 968 969 rc = amd_init_ntb(ndev); 970 if (rc) 971 return rc; 972 973 rc = amd_init_isr(ndev); 974 if (rc) { 975 dev_err(&pdev->dev, "fail to init isr.\n"); 976 return rc; 977 } 978 979 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 980 981 return 0; 982 } 983 984 static void amd_deinit_dev(struct amd_ntb_dev *ndev) 985 { 986 cancel_delayed_work_sync(&ndev->hb_timer); 987 988 ndev_deinit_isr(ndev); 989 } 990 991 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, 992 struct pci_dev *pdev) 993 { 994 int rc; 995 996 pci_set_drvdata(pdev, ndev); 997 998 rc = pci_enable_device(pdev); 999 if (rc) 1000 goto err_pci_enable; 1001 1002 rc = pci_request_regions(pdev, NTB_NAME); 1003 if (rc) 1004 goto err_pci_regions; 1005 1006 pci_set_master(pdev); 1007 1008 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1009 if (rc) { 1010 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 1011 if (rc) 1012 goto err_dma_mask; 1013 dev_warn(&pdev->dev, "Cannot DMA highmem\n"); 1014 } 1015 1016 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1017 if (rc) { 1018 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 1019 if (rc) 1020 goto err_dma_mask; 1021 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); 1022 } 1023 rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev, 1024 dma_get_mask(&pdev->dev)); 1025 if (rc) 1026 goto err_dma_mask; 1027 1028 ndev->self_mmio = pci_iomap(pdev, 0, 0); 1029 if (!ndev->self_mmio) { 1030 rc = -EIO; 1031 goto err_dma_mask; 1032 } 1033 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; 1034 1035 return 0; 1036 1037 err_dma_mask: 1038 pci_clear_master(pdev); 1039 err_pci_regions: 1040 pci_disable_device(pdev); 1041 err_pci_enable: 1042 pci_set_drvdata(pdev, NULL); 1043 return rc; 1044 } 1045 1046 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) 1047 { 1048 struct pci_dev *pdev = ndev->ntb.pdev; 1049 1050 pci_iounmap(pdev, ndev->self_mmio); 1051 1052 pci_clear_master(pdev); 1053 pci_release_regions(pdev); 1054 pci_disable_device(pdev); 1055 pci_set_drvdata(pdev, NULL); 1056 } 1057 1058 static int amd_ntb_pci_probe(struct pci_dev *pdev, 1059 const struct pci_device_id *id) 1060 { 1061 struct amd_ntb_dev *ndev; 1062 int rc, node; 1063 1064 node = dev_to_node(&pdev->dev); 1065 1066 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1067 if (!ndev) { 1068 rc = -ENOMEM; 1069 goto err_ndev; 1070 } 1071 1072 ndev->dev_data = (struct ntb_dev_data *)id->driver_data; 1073 1074 ndev_init_struct(ndev, pdev); 1075 1076 rc = amd_ntb_init_pci(ndev, pdev); 1077 if (rc) 1078 goto err_init_pci; 1079 1080 rc = amd_init_dev(ndev); 1081 if (rc) 1082 goto err_init_dev; 1083 1084 /* write side info */ 1085 amd_init_side_info(ndev); 1086 1087 amd_poll_link(ndev); 1088 1089 ndev_init_debugfs(ndev); 1090 1091 rc = ntb_register_device(&ndev->ntb); 1092 if (rc) 1093 goto err_register; 1094 1095 dev_info(&pdev->dev, "NTB device registered.\n"); 1096 1097 return 0; 1098 1099 err_register: 1100 ndev_deinit_debugfs(ndev); 1101 amd_deinit_dev(ndev); 1102 err_init_dev: 1103 amd_ntb_deinit_pci(ndev); 1104 err_init_pci: 1105 kfree(ndev); 1106 err_ndev: 1107 return rc; 1108 } 1109 1110 static void amd_ntb_pci_remove(struct pci_dev *pdev) 1111 { 1112 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); 1113 1114 ntb_unregister_device(&ndev->ntb); 1115 ndev_deinit_debugfs(ndev); 1116 amd_deinit_side_info(ndev); 1117 amd_deinit_dev(ndev); 1118 amd_ntb_deinit_pci(ndev); 1119 kfree(ndev); 1120 } 1121 1122 static const struct file_operations amd_ntb_debugfs_info = { 1123 .owner = THIS_MODULE, 1124 .open = simple_open, 1125 .read = ndev_debugfs_read, 1126 }; 1127 1128 static const struct ntb_dev_data dev_data[] = { 1129 { /* for device 145b */ 1130 .mw_count = 3, 1131 .mw_idx = 1, 1132 }, 1133 { /* for device 148b */ 1134 .mw_count = 2, 1135 .mw_idx = 2, 1136 }, 1137 }; 1138 1139 static const struct pci_device_id amd_ntb_pci_tbl[] = { 1140 { PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] }, 1141 { PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] }, 1142 { PCI_VDEVICE(HYGON, 0x145b), (kernel_ulong_t)&dev_data[0] }, 1143 { 0, } 1144 }; 1145 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); 1146 1147 static struct pci_driver amd_ntb_pci_driver = { 1148 .name = KBUILD_MODNAME, 1149 .id_table = amd_ntb_pci_tbl, 1150 .probe = amd_ntb_pci_probe, 1151 .remove = amd_ntb_pci_remove, 1152 }; 1153 1154 static int __init amd_ntb_pci_driver_init(void) 1155 { 1156 pr_info("%s %s\n", NTB_DESC, NTB_VER); 1157 1158 if (debugfs_initialized()) 1159 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1160 1161 return pci_register_driver(&amd_ntb_pci_driver); 1162 } 1163 module_init(amd_ntb_pci_driver_init); 1164 1165 static void __exit amd_ntb_pci_driver_exit(void) 1166 { 1167 pci_unregister_driver(&amd_ntb_pci_driver); 1168 debugfs_remove_recursive(debugfs_dir); 1169 } 1170 module_exit(amd_ntb_pci_driver_exit); 1171