1 /* 2 * Cavium ThunderX memory controller kernel module 3 * 4 * This file is subject to the terms and conditions of the GNU General Public 5 * License. See the file "COPYING" in the main directory of this archive 6 * for more details. 7 * 8 * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved. 9 * 10 */ 11 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 #include <linux/edac.h> 15 #include <linux/interrupt.h> 16 #include <linux/string.h> 17 #include <linux/stop_machine.h> 18 #include <linux/delay.h> 19 #include <linux/sizes.h> 20 #include <linux/atomic.h> 21 #include <linux/bitfield.h> 22 #include <linux/circ_buf.h> 23 24 #include <asm/page.h> 25 26 #include "edac_module.h" 27 28 #define phys_to_pfn(phys) (PFN_DOWN(phys)) 29 30 #define THUNDERX_NODE GENMASK(45, 44) 31 32 enum { 33 ERR_CORRECTED = 1, 34 ERR_UNCORRECTED = 2, 35 ERR_UNKNOWN = 3, 36 }; 37 38 struct error_descr { 39 int type; 40 u64 mask; 41 char *descr; 42 }; 43 44 static void decode_register(char *str, size_t size, 45 const struct error_descr *descr, 46 const uint64_t reg) 47 { 48 int ret = 0; 49 50 while (descr->type && descr->mask && descr->descr) { 51 if (reg & descr->mask) { 52 ret = snprintf(str, size, "\n\t%s, %s", 53 descr->type == ERR_CORRECTED ? 54 "Corrected" : "Uncorrected", 55 descr->descr); 56 str += ret; 57 size -= ret; 58 } 59 descr++; 60 } 61 } 62 63 static unsigned long get_bits(unsigned long data, int pos, int width) 64 { 65 return (data >> pos) & ((1 << width) - 1); 66 } 67 68 #define L2C_CTL 0x87E080800000 69 #define L2C_CTL_DISIDXALIAS BIT(0) 70 71 #define PCI_DEVICE_ID_THUNDER_LMC 0xa022 72 73 #define LMC_FADR 0x20 74 #define LMC_FADR_FDIMM(x) ((x >> 37) & 0x1) 75 #define LMC_FADR_FBUNK(x) ((x >> 36) & 0x1) 76 #define LMC_FADR_FBANK(x) ((x >> 32) & 0xf) 77 #define LMC_FADR_FROW(x) ((x >> 14) & 0xffff) 78 #define LMC_FADR_FCOL(x) ((x >> 0) & 0x1fff) 79 80 #define LMC_NXM_FADR 0x28 81 #define LMC_ECC_SYND 0x38 82 83 #define LMC_ECC_PARITY_TEST 0x108 84 85 #define LMC_INT_W1S 0x150 86 87 #define LMC_INT_ENA_W1C 0x158 88 #define LMC_INT_ENA_W1S 0x160 89 90 #define LMC_CONFIG 0x188 91 92 #define LMC_CONFIG_BG2 BIT(62) 93 #define LMC_CONFIG_RANK_ENA BIT(42) 94 #define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF) 95 #define LMC_CONFIG_ROW_LSB(x) (((x) >> 2) & 0x7) 96 97 #define LMC_CONTROL 0x190 98 #define LMC_CONTROL_XOR_BANK BIT(16) 99 100 #define LMC_INT 0x1F0 101 102 #define LMC_INT_DDR_ERR BIT(11) 103 #define LMC_INT_DED_ERR (0xFUL << 5) 104 #define LMC_INT_SEC_ERR (0xFUL << 1) 105 #define LMC_INT_NXM_WR_MASK BIT(0) 106 107 #define LMC_DDR_PLL_CTL 0x258 108 #define LMC_DDR_PLL_CTL_DDR4 BIT(29) 109 110 #define LMC_FADR_SCRAMBLED 0x330 111 112 #define LMC_INT_UE (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \ 113 LMC_INT_NXM_WR_MASK) 114 115 #define LMC_INT_CE (LMC_INT_SEC_ERR) 116 117 static const struct error_descr lmc_errors[] = { 118 { 119 .type = ERR_CORRECTED, 120 .mask = LMC_INT_SEC_ERR, 121 .descr = "Single-bit ECC error", 122 }, 123 { 124 .type = ERR_UNCORRECTED, 125 .mask = LMC_INT_DDR_ERR, 126 .descr = "DDR chip error", 127 }, 128 { 129 .type = ERR_UNCORRECTED, 130 .mask = LMC_INT_DED_ERR, 131 .descr = "Double-bit ECC error", 132 }, 133 { 134 .type = ERR_UNCORRECTED, 135 .mask = LMC_INT_NXM_WR_MASK, 136 .descr = "Non-existent memory write", 137 }, 138 {0, 0, NULL}, 139 }; 140 141 #define LMC_INT_EN_DDR_ERROR_ALERT_ENA BIT(5) 142 #define LMC_INT_EN_DLCRAM_DED_ERR BIT(4) 143 #define LMC_INT_EN_DLCRAM_SEC_ERR BIT(3) 144 #define LMC_INT_INTR_DED_ENA BIT(2) 145 #define LMC_INT_INTR_SEC_ENA BIT(1) 146 #define LMC_INT_INTR_NXM_WR_ENA BIT(0) 147 148 #define LMC_INT_ENA_ALL GENMASK(5, 0) 149 150 #define LMC_DDR_PLL_CTL 0x258 151 #define LMC_DDR_PLL_CTL_DDR4 BIT(29) 152 153 #define LMC_CONTROL 0x190 154 #define LMC_CONTROL_RDIMM BIT(0) 155 156 #define LMC_SCRAM_FADR 0x330 157 158 #define LMC_CHAR_MASK0 0x228 159 #define LMC_CHAR_MASK2 0x238 160 161 #define RING_ENTRIES 8 162 163 struct debugfs_entry { 164 const char *name; 165 umode_t mode; 166 const struct file_operations fops; 167 }; 168 169 struct lmc_err_ctx { 170 u64 reg_int; 171 u64 reg_fadr; 172 u64 reg_nxm_fadr; 173 u64 reg_scram_fadr; 174 u64 reg_ecc_synd; 175 }; 176 177 struct thunderx_lmc { 178 void __iomem *regs; 179 struct pci_dev *pdev; 180 struct msix_entry msix_ent; 181 182 atomic_t ecc_int; 183 184 u64 mask0; 185 u64 mask2; 186 u64 parity_test; 187 u64 node; 188 189 int xbits; 190 int bank_width; 191 int pbank_lsb; 192 int dimm_lsb; 193 int rank_lsb; 194 int bank_lsb; 195 int row_lsb; 196 int col_hi_lsb; 197 198 int xor_bank; 199 int l2c_alias; 200 201 struct page *mem; 202 203 struct lmc_err_ctx err_ctx[RING_ENTRIES]; 204 unsigned long ring_head; 205 unsigned long ring_tail; 206 }; 207 208 #define ring_pos(pos, size) ((pos) & (size - 1)) 209 210 #define DEBUGFS_STRUCT(_name, _mode, _write, _read) \ 211 static struct debugfs_entry debugfs_##_name = { \ 212 .name = __stringify(_name), \ 213 .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \ 214 .fops = { \ 215 .open = simple_open, \ 216 .write = _write, \ 217 .read = _read, \ 218 .llseek = generic_file_llseek, \ 219 }, \ 220 } 221 222 #define DEBUGFS_FIELD_ATTR(_type, _field) \ 223 static ssize_t thunderx_##_type##_##_field##_read(struct file *file, \ 224 char __user *data, \ 225 size_t count, loff_t *ppos) \ 226 { \ 227 struct thunderx_##_type *pdata = file->private_data; \ 228 char buf[20]; \ 229 \ 230 snprintf(buf, count, "0x%016llx", pdata->_field); \ 231 return simple_read_from_buffer(data, count, ppos, \ 232 buf, sizeof(buf)); \ 233 } \ 234 \ 235 static ssize_t thunderx_##_type##_##_field##_write(struct file *file, \ 236 const char __user *data, \ 237 size_t count, loff_t *ppos) \ 238 { \ 239 struct thunderx_##_type *pdata = file->private_data; \ 240 int res; \ 241 \ 242 res = kstrtoull_from_user(data, count, 0, &pdata->_field); \ 243 \ 244 return res ? res : count; \ 245 } \ 246 \ 247 DEBUGFS_STRUCT(_field, 0600, \ 248 thunderx_##_type##_##_field##_write, \ 249 thunderx_##_type##_##_field##_read) \ 250 251 #define DEBUGFS_REG_ATTR(_type, _name, _reg) \ 252 static ssize_t thunderx_##_type##_##_name##_read(struct file *file, \ 253 char __user *data, \ 254 size_t count, loff_t *ppos) \ 255 { \ 256 struct thunderx_##_type *pdata = file->private_data; \ 257 char buf[20]; \ 258 \ 259 sprintf(buf, "0x%016llx", readq(pdata->regs + _reg)); \ 260 return simple_read_from_buffer(data, count, ppos, \ 261 buf, sizeof(buf)); \ 262 } \ 263 \ 264 static ssize_t thunderx_##_type##_##_name##_write(struct file *file, \ 265 const char __user *data, \ 266 size_t count, loff_t *ppos) \ 267 { \ 268 struct thunderx_##_type *pdata = file->private_data; \ 269 u64 val; \ 270 int res; \ 271 \ 272 res = kstrtoull_from_user(data, count, 0, &val); \ 273 \ 274 if (!res) { \ 275 writeq(val, pdata->regs + _reg); \ 276 res = count; \ 277 } \ 278 \ 279 return res; \ 280 } \ 281 \ 282 DEBUGFS_STRUCT(_name, 0600, \ 283 thunderx_##_type##_##_name##_write, \ 284 thunderx_##_type##_##_name##_read) 285 286 #define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field) 287 288 /* 289 * To get an ECC error injected, the following steps are needed: 290 * - Setup the ECC injection by writing the appropriate parameters: 291 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0 292 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2 293 * echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test 294 * - Do the actual injection: 295 * echo 1 > /sys/kernel/debug/<device number>/inject_ecc 296 */ 297 static ssize_t thunderx_lmc_inject_int_write(struct file *file, 298 const char __user *data, 299 size_t count, loff_t *ppos) 300 { 301 struct thunderx_lmc *lmc = file->private_data; 302 u64 val; 303 int res; 304 305 res = kstrtoull_from_user(data, count, 0, &val); 306 307 if (!res) { 308 /* Trigger the interrupt */ 309 writeq(val, lmc->regs + LMC_INT_W1S); 310 res = count; 311 } 312 313 return res; 314 } 315 316 static ssize_t thunderx_lmc_int_read(struct file *file, 317 char __user *data, 318 size_t count, loff_t *ppos) 319 { 320 struct thunderx_lmc *lmc = file->private_data; 321 char buf[20]; 322 u64 lmc_int = readq(lmc->regs + LMC_INT); 323 324 snprintf(buf, sizeof(buf), "0x%016llx", lmc_int); 325 return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf)); 326 } 327 328 #define TEST_PATTERN 0xa5 329 330 static int inject_ecc_fn(void *arg) 331 { 332 struct thunderx_lmc *lmc = arg; 333 uintptr_t addr, phys; 334 unsigned int cline_size = cache_line_size(); 335 const unsigned int lines = PAGE_SIZE / cline_size; 336 unsigned int i, cl_idx; 337 338 addr = (uintptr_t)page_address(lmc->mem); 339 phys = (uintptr_t)page_to_phys(lmc->mem); 340 341 cl_idx = (phys & 0x7f) >> 4; 342 lmc->parity_test &= ~(7ULL << 8); 343 lmc->parity_test |= (cl_idx << 8); 344 345 writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0); 346 writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2); 347 writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST); 348 349 readq(lmc->regs + LMC_CHAR_MASK0); 350 readq(lmc->regs + LMC_CHAR_MASK2); 351 readq(lmc->regs + LMC_ECC_PARITY_TEST); 352 353 for (i = 0; i < lines; i++) { 354 memset((void *)addr, TEST_PATTERN, cline_size); 355 barrier(); 356 357 /* 358 * Flush L1 cachelines to the PoC (L2). 359 * This will cause cacheline eviction to the L2. 360 */ 361 asm volatile("dc civac, %0\n" 362 "dsb sy\n" 363 : : "r"(addr + i * cline_size)); 364 } 365 366 for (i = 0; i < lines; i++) { 367 /* 368 * Flush L2 cachelines to the DRAM. 369 * This will cause cacheline eviction to the DRAM 370 * and ECC corruption according to the masks set. 371 */ 372 __asm__ volatile("sys #0,c11,C1,#2, %0\n" 373 : : "r"(phys + i * cline_size)); 374 } 375 376 for (i = 0; i < lines; i++) { 377 /* 378 * Invalidate L2 cachelines. 379 * The subsequent load will cause cacheline fetch 380 * from the DRAM and an error interrupt 381 */ 382 __asm__ volatile("sys #0,c11,C1,#1, %0" 383 : : "r"(phys + i * cline_size)); 384 } 385 386 for (i = 0; i < lines; i++) { 387 /* 388 * Invalidate L1 cachelines. 389 * The subsequent load will cause cacheline fetch 390 * from the L2 and/or DRAM 391 */ 392 asm volatile("dc ivac, %0\n" 393 "dsb sy\n" 394 : : "r"(addr + i * cline_size)); 395 } 396 397 return 0; 398 } 399 400 static ssize_t thunderx_lmc_inject_ecc_write(struct file *file, 401 const char __user *data, 402 size_t count, loff_t *ppos) 403 { 404 struct thunderx_lmc *lmc = file->private_data; 405 unsigned int cline_size = cache_line_size(); 406 u8 *tmp; 407 void __iomem *addr; 408 unsigned int offs, timeout = 100000; 409 410 atomic_set(&lmc->ecc_int, 0); 411 412 lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0); 413 if (!lmc->mem) 414 return -ENOMEM; 415 416 tmp = kmalloc(cline_size, GFP_KERNEL); 417 if (!tmp) { 418 __free_pages(lmc->mem, 0); 419 return -ENOMEM; 420 } 421 422 addr = page_address(lmc->mem); 423 424 while (!atomic_read(&lmc->ecc_int) && timeout--) { 425 stop_machine(inject_ecc_fn, lmc, NULL); 426 427 for (offs = 0; offs < PAGE_SIZE; offs += cline_size) { 428 /* 429 * Do a load from the previously rigged location 430 * This should generate an error interrupt. 431 */ 432 memcpy(tmp, addr + offs, cline_size); 433 asm volatile("dsb ld\n"); 434 } 435 } 436 437 kfree(tmp); 438 __free_pages(lmc->mem, 0); 439 440 return count; 441 } 442 443 LMC_DEBUGFS_ENT(mask0); 444 LMC_DEBUGFS_ENT(mask2); 445 LMC_DEBUGFS_ENT(parity_test); 446 447 DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL); 448 DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL); 449 DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read); 450 451 static struct debugfs_entry *lmc_dfs_ents[] = { 452 &debugfs_mask0, 453 &debugfs_mask2, 454 &debugfs_parity_test, 455 &debugfs_inject_ecc, 456 &debugfs_inject_int, 457 &debugfs_int_w1c, 458 }; 459 460 static int thunderx_create_debugfs_nodes(struct dentry *parent, 461 struct debugfs_entry *attrs[], 462 void *data, 463 size_t num) 464 { 465 int i; 466 struct dentry *ent; 467 468 if (!IS_ENABLED(CONFIG_EDAC_DEBUG)) 469 return 0; 470 471 if (!parent) 472 return -ENOENT; 473 474 for (i = 0; i < num; i++) { 475 ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode, 476 parent, data, &attrs[i]->fops); 477 478 if (IS_ERR(ent)) 479 break; 480 } 481 482 return i; 483 } 484 485 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc) 486 { 487 phys_addr_t addr = 0; 488 int bank, xbits; 489 490 addr |= lmc->node << 40; 491 addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb; 492 addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb; 493 addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb; 494 addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb; 495 496 bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb; 497 498 if (lmc->xor_bank) 499 bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width); 500 501 addr |= bank << lmc->bank_lsb; 502 503 xbits = PCI_FUNC(lmc->pdev->devfn); 504 505 if (lmc->l2c_alias) 506 xbits ^= get_bits(addr, 20, lmc->xbits) ^ 507 get_bits(addr, 12, lmc->xbits); 508 509 addr |= xbits << 7; 510 511 return addr; 512 } 513 514 static unsigned int thunderx_get_num_lmcs(unsigned int node) 515 { 516 unsigned int number = 0; 517 struct pci_dev *pdev = NULL; 518 519 do { 520 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, 521 PCI_DEVICE_ID_THUNDER_LMC, 522 pdev); 523 if (pdev) { 524 #ifdef CONFIG_NUMA 525 if (pdev->dev.numa_node == node) 526 number++; 527 #else 528 number++; 529 #endif 530 } 531 } while (pdev); 532 533 return number; 534 } 535 536 #define LMC_MESSAGE_SIZE 120 537 #define LMC_OTHER_SIZE (50 * ARRAY_SIZE(lmc_errors)) 538 539 static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id) 540 { 541 struct mem_ctl_info *mci = dev_id; 542 struct thunderx_lmc *lmc = mci->pvt_info; 543 544 unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx)); 545 struct lmc_err_ctx *ctx = &lmc->err_ctx[head]; 546 547 writeq(0, lmc->regs + LMC_CHAR_MASK0); 548 writeq(0, lmc->regs + LMC_CHAR_MASK2); 549 writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST); 550 551 ctx->reg_int = readq(lmc->regs + LMC_INT); 552 ctx->reg_fadr = readq(lmc->regs + LMC_FADR); 553 ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR); 554 ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR); 555 ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND); 556 557 lmc->ring_head++; 558 559 atomic_set(&lmc->ecc_int, 1); 560 561 /* Clear the interrupt */ 562 writeq(ctx->reg_int, lmc->regs + LMC_INT); 563 564 return IRQ_WAKE_THREAD; 565 } 566 567 static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id) 568 { 569 struct mem_ctl_info *mci = dev_id; 570 struct thunderx_lmc *lmc = mci->pvt_info; 571 phys_addr_t phys_addr; 572 573 unsigned long tail; 574 struct lmc_err_ctx *ctx; 575 576 irqreturn_t ret = IRQ_NONE; 577 578 char *msg; 579 char *other; 580 581 msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL); 582 other = kmalloc(LMC_OTHER_SIZE, GFP_KERNEL); 583 584 if (!msg || !other) 585 goto err_free; 586 587 while (CIRC_CNT(lmc->ring_head, lmc->ring_tail, 588 ARRAY_SIZE(lmc->err_ctx))) { 589 tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx)); 590 591 ctx = &lmc->err_ctx[tail]; 592 593 dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n", 594 ctx->reg_int); 595 dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n", 596 ctx->reg_fadr); 597 dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n", 598 ctx->reg_nxm_fadr); 599 dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n", 600 ctx->reg_scram_fadr); 601 dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n", 602 ctx->reg_ecc_synd); 603 604 snprintf(msg, LMC_MESSAGE_SIZE, 605 "DIMM %lld rank %lld bank %lld row %lld col %lld", 606 LMC_FADR_FDIMM(ctx->reg_scram_fadr), 607 LMC_FADR_FBUNK(ctx->reg_scram_fadr), 608 LMC_FADR_FBANK(ctx->reg_scram_fadr), 609 LMC_FADR_FROW(ctx->reg_scram_fadr), 610 LMC_FADR_FCOL(ctx->reg_scram_fadr)); 611 612 decode_register(other, LMC_OTHER_SIZE, lmc_errors, 613 ctx->reg_int); 614 615 phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc); 616 617 if (ctx->reg_int & LMC_INT_UE) 618 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, 619 phys_to_pfn(phys_addr), 620 offset_in_page(phys_addr), 621 0, -1, -1, -1, msg, other); 622 else if (ctx->reg_int & LMC_INT_CE) 623 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1, 624 phys_to_pfn(phys_addr), 625 offset_in_page(phys_addr), 626 0, -1, -1, -1, msg, other); 627 628 lmc->ring_tail++; 629 } 630 631 ret = IRQ_HANDLED; 632 633 err_free: 634 kfree(msg); 635 kfree(other); 636 637 return ret; 638 } 639 640 static const struct pci_device_id thunderx_lmc_pci_tbl[] = { 641 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) }, 642 { 0, }, 643 }; 644 645 static inline int pci_dev_to_mc_idx(struct pci_dev *pdev) 646 { 647 int node = dev_to_node(&pdev->dev); 648 int ret = PCI_FUNC(pdev->devfn); 649 650 ret += max(node, 0) << 3; 651 652 return ret; 653 } 654 655 static int thunderx_lmc_probe(struct pci_dev *pdev, 656 const struct pci_device_id *id) 657 { 658 struct thunderx_lmc *lmc; 659 struct edac_mc_layer layer; 660 struct mem_ctl_info *mci; 661 u64 lmc_control, lmc_ddr_pll_ctl, lmc_config; 662 int ret; 663 u64 lmc_int; 664 void *l2c_ioaddr; 665 666 layer.type = EDAC_MC_LAYER_SLOT; 667 layer.size = 2; 668 layer.is_virt_csrow = false; 669 670 ret = pcim_enable_device(pdev); 671 if (ret) { 672 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret); 673 return ret; 674 } 675 676 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc"); 677 if (ret) { 678 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret); 679 return ret; 680 } 681 682 mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer, 683 sizeof(struct thunderx_lmc)); 684 if (!mci) 685 return -ENOMEM; 686 687 mci->pdev = &pdev->dev; 688 lmc = mci->pvt_info; 689 690 pci_set_drvdata(pdev, mci); 691 692 lmc->regs = pcim_iomap_table(pdev)[0]; 693 694 lmc_control = readq(lmc->regs + LMC_CONTROL); 695 lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL); 696 lmc_config = readq(lmc->regs + LMC_CONFIG); 697 698 if (lmc_control & LMC_CONTROL_RDIMM) { 699 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4, 700 lmc_ddr_pll_ctl) ? 701 MEM_RDDR4 : MEM_RDDR3; 702 } else { 703 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4, 704 lmc_ddr_pll_ctl) ? 705 MEM_DDR4 : MEM_DDR3; 706 } 707 708 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED; 709 mci->edac_cap = EDAC_FLAG_SECDED; 710 711 mci->mod_name = "thunderx-lmc"; 712 mci->ctl_name = "thunderx-lmc"; 713 mci->dev_name = dev_name(&pdev->dev); 714 mci->scrub_mode = SCRUB_NONE; 715 716 lmc->pdev = pdev; 717 lmc->msix_ent.entry = 0; 718 719 lmc->ring_head = 0; 720 lmc->ring_tail = 0; 721 722 ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1); 723 if (ret) { 724 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret); 725 goto err_free; 726 } 727 728 ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector, 729 thunderx_lmc_err_isr, 730 thunderx_lmc_threaded_isr, 0, 731 "[EDAC] ThunderX LMC", mci); 732 if (ret) { 733 dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret); 734 goto err_free; 735 } 736 737 lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0)); 738 739 lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1; 740 lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) && 741 FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3; 742 743 lmc->pbank_lsb = (lmc_config >> 5) & 0xf; 744 lmc->dimm_lsb = 28 + lmc->pbank_lsb + lmc->xbits; 745 lmc->rank_lsb = lmc->dimm_lsb; 746 lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0; 747 lmc->bank_lsb = 7 + lmc->xbits; 748 lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits; 749 750 lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width; 751 752 lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK; 753 754 l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE); 755 if (!l2c_ioaddr) { 756 dev_err(&pdev->dev, "Cannot map L2C_CTL\n"); 757 ret = -ENOMEM; 758 goto err_free; 759 } 760 761 lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS); 762 763 iounmap(l2c_ioaddr); 764 765 ret = edac_mc_add_mc(mci); 766 if (ret) { 767 dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret); 768 goto err_free; 769 } 770 771 lmc_int = readq(lmc->regs + LMC_INT); 772 writeq(lmc_int, lmc->regs + LMC_INT); 773 774 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S); 775 776 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) { 777 ret = thunderx_create_debugfs_nodes(mci->debugfs, 778 lmc_dfs_ents, 779 lmc, 780 ARRAY_SIZE(lmc_dfs_ents)); 781 782 if (ret != ARRAY_SIZE(lmc_dfs_ents)) { 783 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n", 784 ret, ret >= 0 ? " created" : ""); 785 } 786 } 787 788 return 0; 789 790 err_free: 791 pci_set_drvdata(pdev, NULL); 792 edac_mc_free(mci); 793 794 return ret; 795 } 796 797 static void thunderx_lmc_remove(struct pci_dev *pdev) 798 { 799 struct mem_ctl_info *mci = pci_get_drvdata(pdev); 800 struct thunderx_lmc *lmc = mci->pvt_info; 801 802 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C); 803 804 edac_mc_del_mc(&pdev->dev); 805 edac_mc_free(mci); 806 } 807 808 MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl); 809 810 static struct pci_driver thunderx_lmc_driver = { 811 .name = "thunderx_lmc_edac", 812 .probe = thunderx_lmc_probe, 813 .remove = thunderx_lmc_remove, 814 .id_table = thunderx_lmc_pci_tbl, 815 }; 816 817 /*---------------------- OCX driver ---------------------------------*/ 818 819 #define PCI_DEVICE_ID_THUNDER_OCX 0xa013 820 821 #define OCX_LINK_INTS 3 822 #define OCX_INTS (OCX_LINK_INTS + 1) 823 #define OCX_RX_LANES 24 824 #define OCX_RX_LANE_STATS 15 825 826 #define OCX_COM_INT 0x100 827 #define OCX_COM_INT_W1S 0x108 828 #define OCX_COM_INT_ENA_W1S 0x110 829 #define OCX_COM_INT_ENA_W1C 0x118 830 831 #define OCX_COM_IO_BADID BIT(54) 832 #define OCX_COM_MEM_BADID BIT(53) 833 #define OCX_COM_COPR_BADID BIT(52) 834 #define OCX_COM_WIN_REQ_BADID BIT(51) 835 #define OCX_COM_WIN_REQ_TOUT BIT(50) 836 #define OCX_COM_RX_LANE GENMASK(23, 0) 837 838 #define OCX_COM_INT_CE (OCX_COM_IO_BADID | \ 839 OCX_COM_MEM_BADID | \ 840 OCX_COM_COPR_BADID | \ 841 OCX_COM_WIN_REQ_BADID | \ 842 OCX_COM_WIN_REQ_TOUT) 843 844 static const struct error_descr ocx_com_errors[] = { 845 { 846 .type = ERR_CORRECTED, 847 .mask = OCX_COM_IO_BADID, 848 .descr = "Invalid IO transaction node ID", 849 }, 850 { 851 .type = ERR_CORRECTED, 852 .mask = OCX_COM_MEM_BADID, 853 .descr = "Invalid memory transaction node ID", 854 }, 855 { 856 .type = ERR_CORRECTED, 857 .mask = OCX_COM_COPR_BADID, 858 .descr = "Invalid coprocessor transaction node ID", 859 }, 860 { 861 .type = ERR_CORRECTED, 862 .mask = OCX_COM_WIN_REQ_BADID, 863 .descr = "Invalid SLI transaction node ID", 864 }, 865 { 866 .type = ERR_CORRECTED, 867 .mask = OCX_COM_WIN_REQ_TOUT, 868 .descr = "Window/core request timeout", 869 }, 870 {0, 0, NULL}, 871 }; 872 873 #define OCX_COM_LINKX_INT(x) (0x120 + (x) * 8) 874 #define OCX_COM_LINKX_INT_W1S(x) (0x140 + (x) * 8) 875 #define OCX_COM_LINKX_INT_ENA_W1S(x) (0x160 + (x) * 8) 876 #define OCX_COM_LINKX_INT_ENA_W1C(x) (0x180 + (x) * 8) 877 878 #define OCX_COM_LINK_BAD_WORD BIT(13) 879 #define OCX_COM_LINK_ALIGN_FAIL BIT(12) 880 #define OCX_COM_LINK_ALIGN_DONE BIT(11) 881 #define OCX_COM_LINK_UP BIT(10) 882 #define OCX_COM_LINK_STOP BIT(9) 883 #define OCX_COM_LINK_BLK_ERR BIT(8) 884 #define OCX_COM_LINK_REINIT BIT(7) 885 #define OCX_COM_LINK_LNK_DATA BIT(6) 886 #define OCX_COM_LINK_RXFIFO_DBE BIT(5) 887 #define OCX_COM_LINK_RXFIFO_SBE BIT(4) 888 #define OCX_COM_LINK_TXFIFO_DBE BIT(3) 889 #define OCX_COM_LINK_TXFIFO_SBE BIT(2) 890 #define OCX_COM_LINK_REPLAY_DBE BIT(1) 891 #define OCX_COM_LINK_REPLAY_SBE BIT(0) 892 893 static const struct error_descr ocx_com_link_errors[] = { 894 { 895 .type = ERR_CORRECTED, 896 .mask = OCX_COM_LINK_REPLAY_SBE, 897 .descr = "Replay buffer single-bit error", 898 }, 899 { 900 .type = ERR_CORRECTED, 901 .mask = OCX_COM_LINK_TXFIFO_SBE, 902 .descr = "TX FIFO single-bit error", 903 }, 904 { 905 .type = ERR_CORRECTED, 906 .mask = OCX_COM_LINK_RXFIFO_SBE, 907 .descr = "RX FIFO single-bit error", 908 }, 909 { 910 .type = ERR_CORRECTED, 911 .mask = OCX_COM_LINK_BLK_ERR, 912 .descr = "Block code error", 913 }, 914 { 915 .type = ERR_CORRECTED, 916 .mask = OCX_COM_LINK_ALIGN_FAIL, 917 .descr = "Link alignment failure", 918 }, 919 { 920 .type = ERR_CORRECTED, 921 .mask = OCX_COM_LINK_BAD_WORD, 922 .descr = "Bad code word", 923 }, 924 { 925 .type = ERR_UNCORRECTED, 926 .mask = OCX_COM_LINK_REPLAY_DBE, 927 .descr = "Replay buffer double-bit error", 928 }, 929 { 930 .type = ERR_UNCORRECTED, 931 .mask = OCX_COM_LINK_TXFIFO_DBE, 932 .descr = "TX FIFO double-bit error", 933 }, 934 { 935 .type = ERR_UNCORRECTED, 936 .mask = OCX_COM_LINK_RXFIFO_DBE, 937 .descr = "RX FIFO double-bit error", 938 }, 939 { 940 .type = ERR_UNCORRECTED, 941 .mask = OCX_COM_LINK_STOP, 942 .descr = "Link stopped", 943 }, 944 {0, 0, NULL}, 945 }; 946 947 #define OCX_COM_LINK_INT_UE (OCX_COM_LINK_REPLAY_DBE | \ 948 OCX_COM_LINK_TXFIFO_DBE | \ 949 OCX_COM_LINK_RXFIFO_DBE | \ 950 OCX_COM_LINK_STOP) 951 952 #define OCX_COM_LINK_INT_CE (OCX_COM_LINK_REPLAY_SBE | \ 953 OCX_COM_LINK_TXFIFO_SBE | \ 954 OCX_COM_LINK_RXFIFO_SBE | \ 955 OCX_COM_LINK_BLK_ERR | \ 956 OCX_COM_LINK_ALIGN_FAIL | \ 957 OCX_COM_LINK_BAD_WORD) 958 959 #define OCX_LNE_INT(x) (0x8018 + (x) * 0x100) 960 #define OCX_LNE_INT_EN(x) (0x8020 + (x) * 0x100) 961 #define OCX_LNE_BAD_CNT(x) (0x8028 + (x) * 0x100) 962 #define OCX_LNE_CFG(x) (0x8000 + (x) * 0x100) 963 #define OCX_LNE_STAT(x, y) (0x8040 + (x) * 0x100 + (y) * 8) 964 965 #define OCX_LNE_CFG_RX_BDRY_LOCK_DIS BIT(8) 966 #define OCX_LNE_CFG_RX_STAT_WRAP_DIS BIT(2) 967 #define OCX_LNE_CFG_RX_STAT_RDCLR BIT(1) 968 #define OCX_LNE_CFG_RX_STAT_ENA BIT(0) 969 970 971 #define OCX_LANE_BAD_64B67B BIT(8) 972 #define OCX_LANE_DSKEW_FIFO_OVFL BIT(5) 973 #define OCX_LANE_SCRM_SYNC_LOSS BIT(4) 974 #define OCX_LANE_UKWN_CNTL_WORD BIT(3) 975 #define OCX_LANE_CRC32_ERR BIT(2) 976 #define OCX_LANE_BDRY_SYNC_LOSS BIT(1) 977 #define OCX_LANE_SERDES_LOCK_LOSS BIT(0) 978 979 #define OCX_COM_LANE_INT_UE (0) 980 #define OCX_COM_LANE_INT_CE (OCX_LANE_SERDES_LOCK_LOSS | \ 981 OCX_LANE_BDRY_SYNC_LOSS | \ 982 OCX_LANE_CRC32_ERR | \ 983 OCX_LANE_UKWN_CNTL_WORD | \ 984 OCX_LANE_SCRM_SYNC_LOSS | \ 985 OCX_LANE_DSKEW_FIFO_OVFL | \ 986 OCX_LANE_BAD_64B67B) 987 988 static const struct error_descr ocx_lane_errors[] = { 989 { 990 .type = ERR_CORRECTED, 991 .mask = OCX_LANE_SERDES_LOCK_LOSS, 992 .descr = "RX SerDes lock lost", 993 }, 994 { 995 .type = ERR_CORRECTED, 996 .mask = OCX_LANE_BDRY_SYNC_LOSS, 997 .descr = "RX word boundary lost", 998 }, 999 { 1000 .type = ERR_CORRECTED, 1001 .mask = OCX_LANE_CRC32_ERR, 1002 .descr = "CRC32 error", 1003 }, 1004 { 1005 .type = ERR_CORRECTED, 1006 .mask = OCX_LANE_UKWN_CNTL_WORD, 1007 .descr = "Unknown control word", 1008 }, 1009 { 1010 .type = ERR_CORRECTED, 1011 .mask = OCX_LANE_SCRM_SYNC_LOSS, 1012 .descr = "Scrambler synchronization lost", 1013 }, 1014 { 1015 .type = ERR_CORRECTED, 1016 .mask = OCX_LANE_DSKEW_FIFO_OVFL, 1017 .descr = "RX deskew FIFO overflow", 1018 }, 1019 { 1020 .type = ERR_CORRECTED, 1021 .mask = OCX_LANE_BAD_64B67B, 1022 .descr = "Bad 64B/67B codeword", 1023 }, 1024 {0, 0, NULL}, 1025 }; 1026 1027 #define OCX_LNE_INT_ENA_ALL (GENMASK(9, 8) | GENMASK(6, 0)) 1028 #define OCX_COM_INT_ENA_ALL (GENMASK(54, 50) | GENMASK(23, 0)) 1029 #define OCX_COM_LINKX_INT_ENA_ALL (GENMASK(13, 12) | \ 1030 GENMASK(9, 7) | GENMASK(5, 0)) 1031 1032 #define OCX_TLKX_ECC_CTL(x) (0x10018 + (x) * 0x2000) 1033 #define OCX_RLKX_ECC_CTL(x) (0x18018 + (x) * 0x2000) 1034 1035 struct ocx_com_err_ctx { 1036 u64 reg_com_int; 1037 u64 reg_lane_int[OCX_RX_LANES]; 1038 u64 reg_lane_stat11[OCX_RX_LANES]; 1039 }; 1040 1041 struct ocx_link_err_ctx { 1042 u64 reg_com_link_int; 1043 int link; 1044 }; 1045 1046 struct thunderx_ocx { 1047 void __iomem *regs; 1048 int com_link; 1049 struct pci_dev *pdev; 1050 struct edac_device_ctl_info *edac_dev; 1051 1052 struct dentry *debugfs; 1053 struct msix_entry msix_ent[OCX_INTS]; 1054 1055 struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES]; 1056 struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES]; 1057 1058 unsigned long com_ring_head; 1059 unsigned long com_ring_tail; 1060 1061 unsigned long link_ring_head; 1062 unsigned long link_ring_tail; 1063 }; 1064 1065 #define OCX_MESSAGE_SIZE SZ_1K 1066 #define OCX_OTHER_SIZE (50 * ARRAY_SIZE(ocx_com_link_errors)) 1067 1068 /* This handler is threaded */ 1069 static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id) 1070 { 1071 struct msix_entry *msix = irq_id; 1072 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx, 1073 msix_ent[msix->entry]); 1074 1075 int lane; 1076 unsigned long head = ring_pos(ocx->com_ring_head, 1077 ARRAY_SIZE(ocx->com_err_ctx)); 1078 struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head]; 1079 1080 ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT); 1081 1082 for (lane = 0; lane < OCX_RX_LANES; lane++) { 1083 ctx->reg_lane_int[lane] = 1084 readq(ocx->regs + OCX_LNE_INT(lane)); 1085 ctx->reg_lane_stat11[lane] = 1086 readq(ocx->regs + OCX_LNE_STAT(lane, 11)); 1087 1088 writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane)); 1089 } 1090 1091 writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT); 1092 1093 ocx->com_ring_head++; 1094 1095 return IRQ_WAKE_THREAD; 1096 } 1097 1098 static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id) 1099 { 1100 struct msix_entry *msix = irq_id; 1101 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx, 1102 msix_ent[msix->entry]); 1103 1104 irqreturn_t ret = IRQ_NONE; 1105 1106 unsigned long tail; 1107 struct ocx_com_err_ctx *ctx; 1108 int lane; 1109 char *msg; 1110 char *other; 1111 1112 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL); 1113 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL); 1114 1115 if (!msg || !other) 1116 goto err_free; 1117 1118 while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail, 1119 ARRAY_SIZE(ocx->com_err_ctx))) { 1120 tail = ring_pos(ocx->com_ring_tail, 1121 ARRAY_SIZE(ocx->com_err_ctx)); 1122 ctx = &ocx->com_err_ctx[tail]; 1123 1124 snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx", 1125 ocx->edac_dev->ctl_name, ctx->reg_com_int); 1126 1127 decode_register(other, OCX_OTHER_SIZE, 1128 ocx_com_errors, ctx->reg_com_int); 1129 1130 strlcat(msg, other, OCX_MESSAGE_SIZE); 1131 1132 for (lane = 0; lane < OCX_RX_LANES; lane++) 1133 if (ctx->reg_com_int & BIT(lane)) { 1134 snprintf(other, OCX_OTHER_SIZE, 1135 "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx", 1136 lane, ctx->reg_lane_int[lane], 1137 lane, ctx->reg_lane_stat11[lane]); 1138 1139 strlcat(msg, other, OCX_MESSAGE_SIZE); 1140 1141 decode_register(other, OCX_OTHER_SIZE, 1142 ocx_lane_errors, 1143 ctx->reg_lane_int[lane]); 1144 strlcat(msg, other, OCX_MESSAGE_SIZE); 1145 } 1146 1147 if (ctx->reg_com_int & OCX_COM_INT_CE) 1148 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg); 1149 1150 ocx->com_ring_tail++; 1151 } 1152 1153 ret = IRQ_HANDLED; 1154 1155 err_free: 1156 kfree(other); 1157 kfree(msg); 1158 1159 return ret; 1160 } 1161 1162 static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id) 1163 { 1164 struct msix_entry *msix = irq_id; 1165 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx, 1166 msix_ent[msix->entry]); 1167 unsigned long head = ring_pos(ocx->link_ring_head, 1168 ARRAY_SIZE(ocx->link_err_ctx)); 1169 struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head]; 1170 1171 ctx->link = msix->entry; 1172 ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link)); 1173 1174 writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link)); 1175 1176 ocx->link_ring_head++; 1177 1178 return IRQ_WAKE_THREAD; 1179 } 1180 1181 static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id) 1182 { 1183 struct msix_entry *msix = irq_id; 1184 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx, 1185 msix_ent[msix->entry]); 1186 irqreturn_t ret = IRQ_NONE; 1187 unsigned long tail; 1188 struct ocx_link_err_ctx *ctx; 1189 1190 char *msg; 1191 char *other; 1192 1193 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL); 1194 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL); 1195 1196 if (!msg || !other) 1197 goto err_free; 1198 1199 while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail, 1200 ARRAY_SIZE(ocx->link_err_ctx))) { 1201 tail = ring_pos(ocx->link_ring_head, 1202 ARRAY_SIZE(ocx->link_err_ctx)); 1203 1204 ctx = &ocx->link_err_ctx[tail]; 1205 1206 snprintf(msg, OCX_MESSAGE_SIZE, 1207 "%s: OCX_COM_LINK_INT[%d]: %016llx", 1208 ocx->edac_dev->ctl_name, 1209 ctx->link, ctx->reg_com_link_int); 1210 1211 decode_register(other, OCX_OTHER_SIZE, 1212 ocx_com_link_errors, ctx->reg_com_link_int); 1213 1214 strlcat(msg, other, OCX_MESSAGE_SIZE); 1215 1216 if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE) 1217 edac_device_handle_ue(ocx->edac_dev, 0, 0, msg); 1218 else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE) 1219 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg); 1220 1221 ocx->link_ring_tail++; 1222 } 1223 1224 ret = IRQ_HANDLED; 1225 err_free: 1226 kfree(other); 1227 kfree(msg); 1228 1229 return ret; 1230 } 1231 1232 #define OCX_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(ocx, _name, _reg) 1233 1234 OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0)); 1235 OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1)); 1236 OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2)); 1237 1238 OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0)); 1239 OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1)); 1240 OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2)); 1241 1242 OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0)); 1243 OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1)); 1244 OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2)); 1245 1246 OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0)); 1247 OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1)); 1248 OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2)); 1249 OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3)); 1250 OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4)); 1251 OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5)); 1252 OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6)); 1253 OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7)); 1254 1255 OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8)); 1256 OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9)); 1257 OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10)); 1258 OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11)); 1259 OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12)); 1260 OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13)); 1261 OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14)); 1262 OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15)); 1263 1264 OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16)); 1265 OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17)); 1266 OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18)); 1267 OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19)); 1268 OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20)); 1269 OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21)); 1270 OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22)); 1271 OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23)); 1272 1273 OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S); 1274 1275 static struct debugfs_entry *ocx_dfs_ents[] = { 1276 &debugfs_tlk0_ecc_ctl, 1277 &debugfs_tlk1_ecc_ctl, 1278 &debugfs_tlk2_ecc_ctl, 1279 1280 &debugfs_rlk0_ecc_ctl, 1281 &debugfs_rlk1_ecc_ctl, 1282 &debugfs_rlk2_ecc_ctl, 1283 1284 &debugfs_com_link0_int, 1285 &debugfs_com_link1_int, 1286 &debugfs_com_link2_int, 1287 1288 &debugfs_lne00_badcnt, 1289 &debugfs_lne01_badcnt, 1290 &debugfs_lne02_badcnt, 1291 &debugfs_lne03_badcnt, 1292 &debugfs_lne04_badcnt, 1293 &debugfs_lne05_badcnt, 1294 &debugfs_lne06_badcnt, 1295 &debugfs_lne07_badcnt, 1296 &debugfs_lne08_badcnt, 1297 &debugfs_lne09_badcnt, 1298 &debugfs_lne10_badcnt, 1299 &debugfs_lne11_badcnt, 1300 &debugfs_lne12_badcnt, 1301 &debugfs_lne13_badcnt, 1302 &debugfs_lne14_badcnt, 1303 &debugfs_lne15_badcnt, 1304 &debugfs_lne16_badcnt, 1305 &debugfs_lne17_badcnt, 1306 &debugfs_lne18_badcnt, 1307 &debugfs_lne19_badcnt, 1308 &debugfs_lne20_badcnt, 1309 &debugfs_lne21_badcnt, 1310 &debugfs_lne22_badcnt, 1311 &debugfs_lne23_badcnt, 1312 1313 &debugfs_com_int, 1314 }; 1315 1316 static const struct pci_device_id thunderx_ocx_pci_tbl[] = { 1317 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) }, 1318 { 0, }, 1319 }; 1320 1321 static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx) 1322 { 1323 int lane, stat, cfg; 1324 1325 for (lane = 0; lane < OCX_RX_LANES; lane++) { 1326 cfg = readq(ocx->regs + OCX_LNE_CFG(lane)); 1327 cfg |= OCX_LNE_CFG_RX_STAT_RDCLR; 1328 cfg &= ~OCX_LNE_CFG_RX_STAT_ENA; 1329 writeq(cfg, ocx->regs + OCX_LNE_CFG(lane)); 1330 1331 for (stat = 0; stat < OCX_RX_LANE_STATS; stat++) 1332 readq(ocx->regs + OCX_LNE_STAT(lane, stat)); 1333 } 1334 } 1335 1336 static int thunderx_ocx_probe(struct pci_dev *pdev, 1337 const struct pci_device_id *id) 1338 { 1339 struct thunderx_ocx *ocx; 1340 struct edac_device_ctl_info *edac_dev; 1341 char name[32]; 1342 int idx; 1343 int i; 1344 int ret; 1345 u64 reg; 1346 1347 ret = pcim_enable_device(pdev); 1348 if (ret) { 1349 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret); 1350 return ret; 1351 } 1352 1353 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx"); 1354 if (ret) { 1355 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret); 1356 return ret; 1357 } 1358 1359 idx = edac_device_alloc_index(); 1360 snprintf(name, sizeof(name), "OCX%d", idx); 1361 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx), 1362 name, 1, "CCPI", 1, 0, idx); 1363 if (!edac_dev) { 1364 dev_err(&pdev->dev, "Cannot allocate EDAC device\n"); 1365 return -ENOMEM; 1366 } 1367 ocx = edac_dev->pvt_info; 1368 ocx->edac_dev = edac_dev; 1369 ocx->com_ring_head = 0; 1370 ocx->com_ring_tail = 0; 1371 ocx->link_ring_head = 0; 1372 ocx->link_ring_tail = 0; 1373 1374 ocx->regs = pcim_iomap_table(pdev)[0]; 1375 if (!ocx->regs) { 1376 dev_err(&pdev->dev, "Cannot map PCI resources\n"); 1377 ret = -ENODEV; 1378 goto err_free; 1379 } 1380 1381 ocx->pdev = pdev; 1382 1383 for (i = 0; i < OCX_INTS; i++) { 1384 ocx->msix_ent[i].entry = i; 1385 ocx->msix_ent[i].vector = 0; 1386 } 1387 1388 ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS); 1389 if (ret) { 1390 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret); 1391 goto err_free; 1392 } 1393 1394 for (i = 0; i < OCX_INTS; i++) { 1395 ret = devm_request_threaded_irq(&pdev->dev, 1396 ocx->msix_ent[i].vector, 1397 (i == 3) ? 1398 thunderx_ocx_com_isr : 1399 thunderx_ocx_lnk_isr, 1400 (i == 3) ? 1401 thunderx_ocx_com_threaded_isr : 1402 thunderx_ocx_lnk_threaded_isr, 1403 0, "[EDAC] ThunderX OCX", 1404 &ocx->msix_ent[i]); 1405 if (ret) 1406 goto err_free; 1407 } 1408 1409 edac_dev->dev = &pdev->dev; 1410 edac_dev->dev_name = dev_name(&pdev->dev); 1411 edac_dev->mod_name = "thunderx-ocx"; 1412 edac_dev->ctl_name = "thunderx-ocx"; 1413 1414 ret = edac_device_add_device(edac_dev); 1415 if (ret) { 1416 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret); 1417 goto err_free; 1418 } 1419 1420 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) { 1421 ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name); 1422 1423 ret = thunderx_create_debugfs_nodes(ocx->debugfs, 1424 ocx_dfs_ents, 1425 ocx, 1426 ARRAY_SIZE(ocx_dfs_ents)); 1427 if (ret != ARRAY_SIZE(ocx_dfs_ents)) { 1428 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n", 1429 ret, ret >= 0 ? " created" : ""); 1430 } 1431 } 1432 1433 pci_set_drvdata(pdev, edac_dev); 1434 1435 thunderx_ocx_clearstats(ocx); 1436 1437 for (i = 0; i < OCX_RX_LANES; i++) { 1438 writeq(OCX_LNE_INT_ENA_ALL, 1439 ocx->regs + OCX_LNE_INT_EN(i)); 1440 1441 reg = readq(ocx->regs + OCX_LNE_INT(i)); 1442 writeq(reg, ocx->regs + OCX_LNE_INT(i)); 1443 1444 } 1445 1446 for (i = 0; i < OCX_LINK_INTS; i++) { 1447 reg = readq(ocx->regs + OCX_COM_LINKX_INT(i)); 1448 writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i)); 1449 1450 writeq(OCX_COM_LINKX_INT_ENA_ALL, 1451 ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i)); 1452 } 1453 1454 reg = readq(ocx->regs + OCX_COM_INT); 1455 writeq(reg, ocx->regs + OCX_COM_INT); 1456 1457 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S); 1458 1459 return 0; 1460 err_free: 1461 edac_device_free_ctl_info(edac_dev); 1462 1463 return ret; 1464 } 1465 1466 static void thunderx_ocx_remove(struct pci_dev *pdev) 1467 { 1468 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev); 1469 struct thunderx_ocx *ocx = edac_dev->pvt_info; 1470 int i; 1471 1472 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C); 1473 1474 for (i = 0; i < OCX_INTS; i++) { 1475 writeq(OCX_COM_LINKX_INT_ENA_ALL, 1476 ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i)); 1477 } 1478 1479 edac_debugfs_remove_recursive(ocx->debugfs); 1480 1481 edac_device_del_device(&pdev->dev); 1482 edac_device_free_ctl_info(edac_dev); 1483 } 1484 1485 MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl); 1486 1487 static struct pci_driver thunderx_ocx_driver = { 1488 .name = "thunderx_ocx_edac", 1489 .probe = thunderx_ocx_probe, 1490 .remove = thunderx_ocx_remove, 1491 .id_table = thunderx_ocx_pci_tbl, 1492 }; 1493 1494 /*---------------------- L2C driver ---------------------------------*/ 1495 1496 #define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e 1497 #define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f 1498 #define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030 1499 1500 #define L2C_TAD_INT_W1C 0x40000 1501 #define L2C_TAD_INT_W1S 0x40008 1502 1503 #define L2C_TAD_INT_ENA_W1C 0x40020 1504 #define L2C_TAD_INT_ENA_W1S 0x40028 1505 1506 1507 #define L2C_TAD_INT_L2DDBE BIT(1) 1508 #define L2C_TAD_INT_SBFSBE BIT(2) 1509 #define L2C_TAD_INT_SBFDBE BIT(3) 1510 #define L2C_TAD_INT_FBFSBE BIT(4) 1511 #define L2C_TAD_INT_FBFDBE BIT(5) 1512 #define L2C_TAD_INT_TAGDBE BIT(9) 1513 #define L2C_TAD_INT_RDDISLMC BIT(15) 1514 #define L2C_TAD_INT_WRDISLMC BIT(16) 1515 #define L2C_TAD_INT_LFBTO BIT(17) 1516 #define L2C_TAD_INT_GSYNCTO BIT(18) 1517 #define L2C_TAD_INT_RTGSBE BIT(32) 1518 #define L2C_TAD_INT_RTGDBE BIT(33) 1519 #define L2C_TAD_INT_RDDISOCI BIT(34) 1520 #define L2C_TAD_INT_WRDISOCI BIT(35) 1521 1522 #define L2C_TAD_INT_ECC (L2C_TAD_INT_L2DDBE | \ 1523 L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \ 1524 L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE) 1525 1526 #define L2C_TAD_INT_CE (L2C_TAD_INT_SBFSBE | \ 1527 L2C_TAD_INT_FBFSBE) 1528 1529 #define L2C_TAD_INT_UE (L2C_TAD_INT_L2DDBE | \ 1530 L2C_TAD_INT_SBFDBE | \ 1531 L2C_TAD_INT_FBFDBE | \ 1532 L2C_TAD_INT_TAGDBE | \ 1533 L2C_TAD_INT_RTGDBE | \ 1534 L2C_TAD_INT_WRDISOCI | \ 1535 L2C_TAD_INT_RDDISOCI | \ 1536 L2C_TAD_INT_WRDISLMC | \ 1537 L2C_TAD_INT_RDDISLMC | \ 1538 L2C_TAD_INT_LFBTO | \ 1539 L2C_TAD_INT_GSYNCTO) 1540 1541 static const struct error_descr l2_tad_errors[] = { 1542 { 1543 .type = ERR_CORRECTED, 1544 .mask = L2C_TAD_INT_SBFSBE, 1545 .descr = "SBF single-bit error", 1546 }, 1547 { 1548 .type = ERR_CORRECTED, 1549 .mask = L2C_TAD_INT_FBFSBE, 1550 .descr = "FBF single-bit error", 1551 }, 1552 { 1553 .type = ERR_UNCORRECTED, 1554 .mask = L2C_TAD_INT_L2DDBE, 1555 .descr = "L2D double-bit error", 1556 }, 1557 { 1558 .type = ERR_UNCORRECTED, 1559 .mask = L2C_TAD_INT_SBFDBE, 1560 .descr = "SBF double-bit error", 1561 }, 1562 { 1563 .type = ERR_UNCORRECTED, 1564 .mask = L2C_TAD_INT_FBFDBE, 1565 .descr = "FBF double-bit error", 1566 }, 1567 { 1568 .type = ERR_UNCORRECTED, 1569 .mask = L2C_TAD_INT_TAGDBE, 1570 .descr = "TAG double-bit error", 1571 }, 1572 { 1573 .type = ERR_UNCORRECTED, 1574 .mask = L2C_TAD_INT_RTGDBE, 1575 .descr = "RTG double-bit error", 1576 }, 1577 { 1578 .type = ERR_UNCORRECTED, 1579 .mask = L2C_TAD_INT_WRDISOCI, 1580 .descr = "Write to a disabled CCPI", 1581 }, 1582 { 1583 .type = ERR_UNCORRECTED, 1584 .mask = L2C_TAD_INT_RDDISOCI, 1585 .descr = "Read from a disabled CCPI", 1586 }, 1587 { 1588 .type = ERR_UNCORRECTED, 1589 .mask = L2C_TAD_INT_WRDISLMC, 1590 .descr = "Write to a disabled LMC", 1591 }, 1592 { 1593 .type = ERR_UNCORRECTED, 1594 .mask = L2C_TAD_INT_RDDISLMC, 1595 .descr = "Read from a disabled LMC", 1596 }, 1597 { 1598 .type = ERR_UNCORRECTED, 1599 .mask = L2C_TAD_INT_LFBTO, 1600 .descr = "LFB entry timeout", 1601 }, 1602 { 1603 .type = ERR_UNCORRECTED, 1604 .mask = L2C_TAD_INT_GSYNCTO, 1605 .descr = "Global sync CCPI timeout", 1606 }, 1607 {0, 0, NULL}, 1608 }; 1609 1610 #define L2C_TAD_INT_TAG (L2C_TAD_INT_TAGDBE) 1611 1612 #define L2C_TAD_INT_RTG (L2C_TAD_INT_RTGDBE) 1613 1614 #define L2C_TAD_INT_DISLMC (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC) 1615 1616 #define L2C_TAD_INT_DISOCI (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI) 1617 1618 #define L2C_TAD_INT_ENA_ALL (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \ 1619 L2C_TAD_INT_RTG | \ 1620 L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \ 1621 L2C_TAD_INT_LFBTO) 1622 1623 #define L2C_TAD_TIMETWO 0x50000 1624 #define L2C_TAD_TIMEOUT 0x50100 1625 #define L2C_TAD_ERR 0x60000 1626 #define L2C_TAD_TQD_ERR 0x60100 1627 #define L2C_TAD_TTG_ERR 0x60200 1628 1629 1630 #define L2C_CBC_INT_W1C 0x60000 1631 1632 #define L2C_CBC_INT_RSDSBE BIT(0) 1633 #define L2C_CBC_INT_RSDDBE BIT(1) 1634 1635 #define L2C_CBC_INT_RSD (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE) 1636 1637 #define L2C_CBC_INT_MIBSBE BIT(4) 1638 #define L2C_CBC_INT_MIBDBE BIT(5) 1639 1640 #define L2C_CBC_INT_MIB (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE) 1641 1642 #define L2C_CBC_INT_IORDDISOCI BIT(6) 1643 #define L2C_CBC_INT_IOWRDISOCI BIT(7) 1644 1645 #define L2C_CBC_INT_IODISOCI (L2C_CBC_INT_IORDDISOCI | \ 1646 L2C_CBC_INT_IOWRDISOCI) 1647 1648 #define L2C_CBC_INT_CE (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE) 1649 #define L2C_CBC_INT_UE (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE) 1650 1651 1652 static const struct error_descr l2_cbc_errors[] = { 1653 { 1654 .type = ERR_CORRECTED, 1655 .mask = L2C_CBC_INT_RSDSBE, 1656 .descr = "RSD single-bit error", 1657 }, 1658 { 1659 .type = ERR_CORRECTED, 1660 .mask = L2C_CBC_INT_MIBSBE, 1661 .descr = "MIB single-bit error", 1662 }, 1663 { 1664 .type = ERR_UNCORRECTED, 1665 .mask = L2C_CBC_INT_RSDDBE, 1666 .descr = "RSD double-bit error", 1667 }, 1668 { 1669 .type = ERR_UNCORRECTED, 1670 .mask = L2C_CBC_INT_MIBDBE, 1671 .descr = "MIB double-bit error", 1672 }, 1673 { 1674 .type = ERR_UNCORRECTED, 1675 .mask = L2C_CBC_INT_IORDDISOCI, 1676 .descr = "Read from a disabled CCPI", 1677 }, 1678 { 1679 .type = ERR_UNCORRECTED, 1680 .mask = L2C_CBC_INT_IOWRDISOCI, 1681 .descr = "Write to a disabled CCPI", 1682 }, 1683 {0, 0, NULL}, 1684 }; 1685 1686 #define L2C_CBC_INT_W1S 0x60008 1687 #define L2C_CBC_INT_ENA_W1C 0x60020 1688 1689 #define L2C_CBC_INT_ENA_ALL (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \ 1690 L2C_CBC_INT_IODISOCI) 1691 1692 #define L2C_CBC_INT_ENA_W1S 0x60028 1693 1694 #define L2C_CBC_IODISOCIERR 0x80008 1695 #define L2C_CBC_IOCERR 0x80010 1696 #define L2C_CBC_RSDERR 0x80018 1697 #define L2C_CBC_MIBERR 0x80020 1698 1699 1700 #define L2C_MCI_INT_W1C 0x0 1701 1702 #define L2C_MCI_INT_VBFSBE BIT(0) 1703 #define L2C_MCI_INT_VBFDBE BIT(1) 1704 1705 static const struct error_descr l2_mci_errors[] = { 1706 { 1707 .type = ERR_CORRECTED, 1708 .mask = L2C_MCI_INT_VBFSBE, 1709 .descr = "VBF single-bit error", 1710 }, 1711 { 1712 .type = ERR_UNCORRECTED, 1713 .mask = L2C_MCI_INT_VBFDBE, 1714 .descr = "VBF double-bit error", 1715 }, 1716 {0, 0, NULL}, 1717 }; 1718 1719 #define L2C_MCI_INT_W1S 0x8 1720 #define L2C_MCI_INT_ENA_W1C 0x20 1721 1722 #define L2C_MCI_INT_ENA_ALL (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE) 1723 1724 #define L2C_MCI_INT_ENA_W1S 0x28 1725 1726 #define L2C_MCI_ERR 0x10000 1727 1728 #define L2C_MESSAGE_SIZE SZ_1K 1729 #define L2C_OTHER_SIZE (50 * ARRAY_SIZE(l2_tad_errors)) 1730 1731 struct l2c_err_ctx { 1732 char *reg_ext_name; 1733 u64 reg_int; 1734 u64 reg_ext; 1735 }; 1736 1737 struct thunderx_l2c { 1738 void __iomem *regs; 1739 struct pci_dev *pdev; 1740 struct edac_device_ctl_info *edac_dev; 1741 1742 struct dentry *debugfs; 1743 1744 int index; 1745 1746 struct msix_entry msix_ent; 1747 1748 struct l2c_err_ctx err_ctx[RING_ENTRIES]; 1749 unsigned long ring_head; 1750 unsigned long ring_tail; 1751 }; 1752 1753 static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id) 1754 { 1755 struct msix_entry *msix = irq_id; 1756 struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c, 1757 msix_ent); 1758 1759 unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx)); 1760 struct l2c_err_ctx *ctx = &tad->err_ctx[head]; 1761 1762 ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C); 1763 1764 if (ctx->reg_int & L2C_TAD_INT_ECC) { 1765 ctx->reg_ext_name = "TQD_ERR"; 1766 ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR); 1767 } else if (ctx->reg_int & L2C_TAD_INT_TAG) { 1768 ctx->reg_ext_name = "TTG_ERR"; 1769 ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR); 1770 } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) { 1771 ctx->reg_ext_name = "TIMEOUT"; 1772 ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT); 1773 } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) { 1774 ctx->reg_ext_name = "ERR"; 1775 ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR); 1776 } 1777 1778 writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C); 1779 1780 tad->ring_head++; 1781 1782 return IRQ_WAKE_THREAD; 1783 } 1784 1785 static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id) 1786 { 1787 struct msix_entry *msix = irq_id; 1788 struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c, 1789 msix_ent); 1790 1791 unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx)); 1792 struct l2c_err_ctx *ctx = &cbc->err_ctx[head]; 1793 1794 ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C); 1795 1796 if (ctx->reg_int & L2C_CBC_INT_RSD) { 1797 ctx->reg_ext_name = "RSDERR"; 1798 ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR); 1799 } else if (ctx->reg_int & L2C_CBC_INT_MIB) { 1800 ctx->reg_ext_name = "MIBERR"; 1801 ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR); 1802 } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) { 1803 ctx->reg_ext_name = "IODISOCIERR"; 1804 ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR); 1805 } 1806 1807 writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C); 1808 1809 cbc->ring_head++; 1810 1811 return IRQ_WAKE_THREAD; 1812 } 1813 1814 static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id) 1815 { 1816 struct msix_entry *msix = irq_id; 1817 struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c, 1818 msix_ent); 1819 1820 unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx)); 1821 struct l2c_err_ctx *ctx = &mci->err_ctx[head]; 1822 1823 ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C); 1824 ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR); 1825 1826 writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C); 1827 1828 ctx->reg_ext_name = "ERR"; 1829 1830 mci->ring_head++; 1831 1832 return IRQ_WAKE_THREAD; 1833 } 1834 1835 static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id) 1836 { 1837 struct msix_entry *msix = irq_id; 1838 struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c, 1839 msix_ent); 1840 1841 unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx)); 1842 struct l2c_err_ctx *ctx = &l2c->err_ctx[tail]; 1843 irqreturn_t ret = IRQ_NONE; 1844 1845 u64 mask_ue, mask_ce; 1846 const struct error_descr *l2_errors; 1847 char *reg_int_name; 1848 1849 char *msg; 1850 char *other; 1851 1852 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL); 1853 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL); 1854 1855 if (!msg || !other) 1856 goto err_free; 1857 1858 switch (l2c->pdev->device) { 1859 case PCI_DEVICE_ID_THUNDER_L2C_TAD: 1860 reg_int_name = "L2C_TAD_INT"; 1861 mask_ue = L2C_TAD_INT_UE; 1862 mask_ce = L2C_TAD_INT_CE; 1863 l2_errors = l2_tad_errors; 1864 break; 1865 case PCI_DEVICE_ID_THUNDER_L2C_CBC: 1866 reg_int_name = "L2C_CBC_INT"; 1867 mask_ue = L2C_CBC_INT_UE; 1868 mask_ce = L2C_CBC_INT_CE; 1869 l2_errors = l2_cbc_errors; 1870 break; 1871 case PCI_DEVICE_ID_THUNDER_L2C_MCI: 1872 reg_int_name = "L2C_MCI_INT"; 1873 mask_ue = L2C_MCI_INT_VBFDBE; 1874 mask_ce = L2C_MCI_INT_VBFSBE; 1875 l2_errors = l2_mci_errors; 1876 break; 1877 default: 1878 dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n", 1879 l2c->pdev->device); 1880 goto err_free; 1881 } 1882 1883 while (CIRC_CNT(l2c->ring_head, l2c->ring_tail, 1884 ARRAY_SIZE(l2c->err_ctx))) { 1885 snprintf(msg, L2C_MESSAGE_SIZE, 1886 "%s: %s: %016llx, %s: %016llx", 1887 l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int, 1888 ctx->reg_ext_name, ctx->reg_ext); 1889 1890 decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int); 1891 1892 strlcat(msg, other, L2C_MESSAGE_SIZE); 1893 1894 if (ctx->reg_int & mask_ue) 1895 edac_device_handle_ue(l2c->edac_dev, 0, 0, msg); 1896 else if (ctx->reg_int & mask_ce) 1897 edac_device_handle_ce(l2c->edac_dev, 0, 0, msg); 1898 1899 l2c->ring_tail++; 1900 } 1901 1902 ret = IRQ_HANDLED; 1903 1904 err_free: 1905 kfree(other); 1906 kfree(msg); 1907 1908 return ret; 1909 } 1910 1911 #define L2C_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(l2c, _name, _reg) 1912 1913 L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S); 1914 1915 static struct debugfs_entry *l2c_tad_dfs_ents[] = { 1916 &debugfs_tad_int, 1917 }; 1918 1919 L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S); 1920 1921 static struct debugfs_entry *l2c_cbc_dfs_ents[] = { 1922 &debugfs_cbc_int, 1923 }; 1924 1925 L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S); 1926 1927 static struct debugfs_entry *l2c_mci_dfs_ents[] = { 1928 &debugfs_mci_int, 1929 }; 1930 1931 static const struct pci_device_id thunderx_l2c_pci_tbl[] = { 1932 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), }, 1933 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), }, 1934 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), }, 1935 { 0, }, 1936 }; 1937 1938 static int thunderx_l2c_probe(struct pci_dev *pdev, 1939 const struct pci_device_id *id) 1940 { 1941 struct thunderx_l2c *l2c; 1942 struct edac_device_ctl_info *edac_dev; 1943 struct debugfs_entry **l2c_devattr; 1944 size_t dfs_entries; 1945 irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL; 1946 char name[32]; 1947 const char *fmt; 1948 u64 reg_en_offs, reg_en_mask; 1949 int idx; 1950 int ret; 1951 1952 ret = pcim_enable_device(pdev); 1953 if (ret) { 1954 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret); 1955 return ret; 1956 } 1957 1958 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c"); 1959 if (ret) { 1960 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret); 1961 return ret; 1962 } 1963 1964 switch (pdev->device) { 1965 case PCI_DEVICE_ID_THUNDER_L2C_TAD: 1966 thunderx_l2c_isr = thunderx_l2c_tad_isr; 1967 l2c_devattr = l2c_tad_dfs_ents; 1968 dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents); 1969 fmt = "L2C-TAD%d"; 1970 reg_en_offs = L2C_TAD_INT_ENA_W1S; 1971 reg_en_mask = L2C_TAD_INT_ENA_ALL; 1972 break; 1973 case PCI_DEVICE_ID_THUNDER_L2C_CBC: 1974 thunderx_l2c_isr = thunderx_l2c_cbc_isr; 1975 l2c_devattr = l2c_cbc_dfs_ents; 1976 dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents); 1977 fmt = "L2C-CBC%d"; 1978 reg_en_offs = L2C_CBC_INT_ENA_W1S; 1979 reg_en_mask = L2C_CBC_INT_ENA_ALL; 1980 break; 1981 case PCI_DEVICE_ID_THUNDER_L2C_MCI: 1982 thunderx_l2c_isr = thunderx_l2c_mci_isr; 1983 l2c_devattr = l2c_mci_dfs_ents; 1984 dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents); 1985 fmt = "L2C-MCI%d"; 1986 reg_en_offs = L2C_MCI_INT_ENA_W1S; 1987 reg_en_mask = L2C_MCI_INT_ENA_ALL; 1988 break; 1989 default: 1990 //Should never ever get here 1991 dev_err(&pdev->dev, "Unsupported PCI device: %04x\n", 1992 pdev->device); 1993 return -EINVAL; 1994 } 1995 1996 idx = edac_device_alloc_index(); 1997 snprintf(name, sizeof(name), fmt, idx); 1998 1999 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c), 2000 name, 1, "L2C", 1, 0, idx); 2001 if (!edac_dev) { 2002 dev_err(&pdev->dev, "Cannot allocate EDAC device\n"); 2003 return -ENOMEM; 2004 } 2005 2006 l2c = edac_dev->pvt_info; 2007 l2c->edac_dev = edac_dev; 2008 2009 l2c->regs = pcim_iomap_table(pdev)[0]; 2010 if (!l2c->regs) { 2011 dev_err(&pdev->dev, "Cannot map PCI resources\n"); 2012 ret = -ENODEV; 2013 goto err_free; 2014 } 2015 2016 l2c->pdev = pdev; 2017 2018 l2c->ring_head = 0; 2019 l2c->ring_tail = 0; 2020 2021 l2c->msix_ent.entry = 0; 2022 l2c->msix_ent.vector = 0; 2023 2024 ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1); 2025 if (ret) { 2026 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret); 2027 goto err_free; 2028 } 2029 2030 ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector, 2031 thunderx_l2c_isr, 2032 thunderx_l2c_threaded_isr, 2033 0, "[EDAC] ThunderX L2C", 2034 &l2c->msix_ent); 2035 if (ret) 2036 goto err_free; 2037 2038 edac_dev->dev = &pdev->dev; 2039 edac_dev->dev_name = dev_name(&pdev->dev); 2040 edac_dev->mod_name = "thunderx-l2c"; 2041 edac_dev->ctl_name = "thunderx-l2c"; 2042 2043 ret = edac_device_add_device(edac_dev); 2044 if (ret) { 2045 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret); 2046 goto err_free; 2047 } 2048 2049 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) { 2050 l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name); 2051 2052 ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr, 2053 l2c, dfs_entries); 2054 2055 if (ret != dfs_entries) { 2056 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n", 2057 ret, ret >= 0 ? " created" : ""); 2058 } 2059 } 2060 2061 pci_set_drvdata(pdev, edac_dev); 2062 2063 writeq(reg_en_mask, l2c->regs + reg_en_offs); 2064 2065 return 0; 2066 2067 err_free: 2068 edac_device_free_ctl_info(edac_dev); 2069 2070 return ret; 2071 } 2072 2073 static void thunderx_l2c_remove(struct pci_dev *pdev) 2074 { 2075 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev); 2076 struct thunderx_l2c *l2c = edac_dev->pvt_info; 2077 2078 switch (pdev->device) { 2079 case PCI_DEVICE_ID_THUNDER_L2C_TAD: 2080 writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C); 2081 break; 2082 case PCI_DEVICE_ID_THUNDER_L2C_CBC: 2083 writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C); 2084 break; 2085 case PCI_DEVICE_ID_THUNDER_L2C_MCI: 2086 writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C); 2087 break; 2088 } 2089 2090 edac_debugfs_remove_recursive(l2c->debugfs); 2091 2092 edac_device_del_device(&pdev->dev); 2093 edac_device_free_ctl_info(edac_dev); 2094 } 2095 2096 MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl); 2097 2098 static struct pci_driver thunderx_l2c_driver = { 2099 .name = "thunderx_l2c_edac", 2100 .probe = thunderx_l2c_probe, 2101 .remove = thunderx_l2c_remove, 2102 .id_table = thunderx_l2c_pci_tbl, 2103 }; 2104 2105 static int __init thunderx_edac_init(void) 2106 { 2107 int rc = 0; 2108 2109 if (ghes_get_devices()) 2110 return -EBUSY; 2111 2112 rc = pci_register_driver(&thunderx_lmc_driver); 2113 if (rc) 2114 return rc; 2115 2116 rc = pci_register_driver(&thunderx_ocx_driver); 2117 if (rc) 2118 goto err_lmc; 2119 2120 rc = pci_register_driver(&thunderx_l2c_driver); 2121 if (rc) 2122 goto err_ocx; 2123 2124 return rc; 2125 err_ocx: 2126 pci_unregister_driver(&thunderx_ocx_driver); 2127 err_lmc: 2128 pci_unregister_driver(&thunderx_lmc_driver); 2129 2130 return rc; 2131 } 2132 2133 static void __exit thunderx_edac_exit(void) 2134 { 2135 pci_unregister_driver(&thunderx_l2c_driver); 2136 pci_unregister_driver(&thunderx_ocx_driver); 2137 pci_unregister_driver(&thunderx_lmc_driver); 2138 2139 } 2140 2141 module_init(thunderx_edac_init); 2142 module_exit(thunderx_edac_exit); 2143 2144 MODULE_LICENSE("GPL v2"); 2145 MODULE_AUTHOR("Cavium, Inc."); 2146 MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX"); 2147