1 /*
2 * Cavium ThunderX memory controller kernel module
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
9 *
10 */
11
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/edac.h>
15 #include <linux/interrupt.h>
16 #include <linux/string.h>
17 #include <linux/stop_machine.h>
18 #include <linux/delay.h>
19 #include <linux/sizes.h>
20 #include <linux/atomic.h>
21 #include <linux/bitfield.h>
22 #include <linux/circ_buf.h>
23
24 #include <asm/page.h>
25
26 #include "edac_module.h"
27
28 #define phys_to_pfn(phys) (PFN_DOWN(phys))
29
30 #define THUNDERX_NODE GENMASK(45, 44)
31
32 enum {
33 ERR_CORRECTED = 1,
34 ERR_UNCORRECTED = 2,
35 ERR_UNKNOWN = 3,
36 };
37
38 struct error_descr {
39 int type;
40 u64 mask;
41 char *descr;
42 };
43
decode_register(char * str,size_t size,const struct error_descr * descr,const uint64_t reg)44 static void decode_register(char *str, size_t size,
45 const struct error_descr *descr,
46 const uint64_t reg)
47 {
48 int ret = 0;
49
50 while (descr->type && descr->mask && descr->descr) {
51 if (reg & descr->mask) {
52 ret = snprintf(str, size, "\n\t%s, %s",
53 descr->type == ERR_CORRECTED ?
54 "Corrected" : "Uncorrected",
55 descr->descr);
56 str += ret;
57 size -= ret;
58 }
59 descr++;
60 }
61 }
62
get_bits(unsigned long data,int pos,int width)63 static unsigned long get_bits(unsigned long data, int pos, int width)
64 {
65 return (data >> pos) & ((1 << width) - 1);
66 }
67
68 #define L2C_CTL 0x87E080800000
69 #define L2C_CTL_DISIDXALIAS BIT(0)
70
71 #define PCI_DEVICE_ID_THUNDER_LMC 0xa022
72
73 #define LMC_FADR 0x20
74 #define LMC_FADR_FDIMM(x) ((x >> 37) & 0x1)
75 #define LMC_FADR_FBUNK(x) ((x >> 36) & 0x1)
76 #define LMC_FADR_FBANK(x) ((x >> 32) & 0xf)
77 #define LMC_FADR_FROW(x) ((x >> 14) & 0xffff)
78 #define LMC_FADR_FCOL(x) ((x >> 0) & 0x1fff)
79
80 #define LMC_NXM_FADR 0x28
81 #define LMC_ECC_SYND 0x38
82
83 #define LMC_ECC_PARITY_TEST 0x108
84
85 #define LMC_INT_W1S 0x150
86
87 #define LMC_INT_ENA_W1C 0x158
88 #define LMC_INT_ENA_W1S 0x160
89
90 #define LMC_CONFIG 0x188
91
92 #define LMC_CONFIG_BG2 BIT(62)
93 #define LMC_CONFIG_RANK_ENA BIT(42)
94 #define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF)
95 #define LMC_CONFIG_ROW_LSB(x) (((x) >> 2) & 0x7)
96
97 #define LMC_CONTROL 0x190
98 #define LMC_CONTROL_XOR_BANK BIT(16)
99
100 #define LMC_INT 0x1F0
101
102 #define LMC_INT_DDR_ERR BIT(11)
103 #define LMC_INT_DED_ERR (0xFUL << 5)
104 #define LMC_INT_SEC_ERR (0xFUL << 1)
105 #define LMC_INT_NXM_WR_MASK BIT(0)
106
107 #define LMC_DDR_PLL_CTL 0x258
108 #define LMC_DDR_PLL_CTL_DDR4 BIT(29)
109
110 #define LMC_FADR_SCRAMBLED 0x330
111
112 #define LMC_INT_UE (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
113 LMC_INT_NXM_WR_MASK)
114
115 #define LMC_INT_CE (LMC_INT_SEC_ERR)
116
117 static const struct error_descr lmc_errors[] = {
118 {
119 .type = ERR_CORRECTED,
120 .mask = LMC_INT_SEC_ERR,
121 .descr = "Single-bit ECC error",
122 },
123 {
124 .type = ERR_UNCORRECTED,
125 .mask = LMC_INT_DDR_ERR,
126 .descr = "DDR chip error",
127 },
128 {
129 .type = ERR_UNCORRECTED,
130 .mask = LMC_INT_DED_ERR,
131 .descr = "Double-bit ECC error",
132 },
133 {
134 .type = ERR_UNCORRECTED,
135 .mask = LMC_INT_NXM_WR_MASK,
136 .descr = "Non-existent memory write",
137 },
138 {0, 0, NULL},
139 };
140
141 #define LMC_INT_EN_DDR_ERROR_ALERT_ENA BIT(5)
142 #define LMC_INT_EN_DLCRAM_DED_ERR BIT(4)
143 #define LMC_INT_EN_DLCRAM_SEC_ERR BIT(3)
144 #define LMC_INT_INTR_DED_ENA BIT(2)
145 #define LMC_INT_INTR_SEC_ENA BIT(1)
146 #define LMC_INT_INTR_NXM_WR_ENA BIT(0)
147
148 #define LMC_INT_ENA_ALL GENMASK(5, 0)
149
150 #define LMC_DDR_PLL_CTL 0x258
151 #define LMC_DDR_PLL_CTL_DDR4 BIT(29)
152
153 #define LMC_CONTROL 0x190
154 #define LMC_CONTROL_RDIMM BIT(0)
155
156 #define LMC_SCRAM_FADR 0x330
157
158 #define LMC_CHAR_MASK0 0x228
159 #define LMC_CHAR_MASK2 0x238
160
161 #define RING_ENTRIES 8
162
163 struct debugfs_entry {
164 const char *name;
165 umode_t mode;
166 const struct file_operations fops;
167 };
168
169 struct lmc_err_ctx {
170 u64 reg_int;
171 u64 reg_fadr;
172 u64 reg_nxm_fadr;
173 u64 reg_scram_fadr;
174 u64 reg_ecc_synd;
175 };
176
177 struct thunderx_lmc {
178 void __iomem *regs;
179 struct pci_dev *pdev;
180 struct msix_entry msix_ent;
181
182 atomic_t ecc_int;
183
184 u64 mask0;
185 u64 mask2;
186 u64 parity_test;
187 u64 node;
188
189 int xbits;
190 int bank_width;
191 int pbank_lsb;
192 int dimm_lsb;
193 int rank_lsb;
194 int bank_lsb;
195 int row_lsb;
196 int col_hi_lsb;
197
198 int xor_bank;
199 int l2c_alias;
200
201 struct page *mem;
202
203 struct lmc_err_ctx err_ctx[RING_ENTRIES];
204 unsigned long ring_head;
205 unsigned long ring_tail;
206 };
207
208 #define ring_pos(pos, size) ((pos) & (size - 1))
209
210 #define DEBUGFS_STRUCT(_name, _mode, _write, _read) \
211 static struct debugfs_entry debugfs_##_name = { \
212 .name = __stringify(_name), \
213 .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \
214 .fops = { \
215 .open = simple_open, \
216 .write = _write, \
217 .read = _read, \
218 .llseek = generic_file_llseek, \
219 }, \
220 }
221
222 #define DEBUGFS_FIELD_ATTR(_type, _field) \
223 static ssize_t thunderx_##_type##_##_field##_read(struct file *file, \
224 char __user *data, \
225 size_t count, loff_t *ppos) \
226 { \
227 struct thunderx_##_type *pdata = file->private_data; \
228 char buf[20]; \
229 \
230 snprintf(buf, count, "0x%016llx", pdata->_field); \
231 return simple_read_from_buffer(data, count, ppos, \
232 buf, sizeof(buf)); \
233 } \
234 \
235 static ssize_t thunderx_##_type##_##_field##_write(struct file *file, \
236 const char __user *data, \
237 size_t count, loff_t *ppos) \
238 { \
239 struct thunderx_##_type *pdata = file->private_data; \
240 int res; \
241 \
242 res = kstrtoull_from_user(data, count, 0, &pdata->_field); \
243 \
244 return res ? res : count; \
245 } \
246 \
247 DEBUGFS_STRUCT(_field, 0600, \
248 thunderx_##_type##_##_field##_write, \
249 thunderx_##_type##_##_field##_read) \
250
251 #define DEBUGFS_REG_ATTR(_type, _name, _reg) \
252 static ssize_t thunderx_##_type##_##_name##_read(struct file *file, \
253 char __user *data, \
254 size_t count, loff_t *ppos) \
255 { \
256 struct thunderx_##_type *pdata = file->private_data; \
257 char buf[20]; \
258 \
259 sprintf(buf, "0x%016llx", readq(pdata->regs + _reg)); \
260 return simple_read_from_buffer(data, count, ppos, \
261 buf, sizeof(buf)); \
262 } \
263 \
264 static ssize_t thunderx_##_type##_##_name##_write(struct file *file, \
265 const char __user *data, \
266 size_t count, loff_t *ppos) \
267 { \
268 struct thunderx_##_type *pdata = file->private_data; \
269 u64 val; \
270 int res; \
271 \
272 res = kstrtoull_from_user(data, count, 0, &val); \
273 \
274 if (!res) { \
275 writeq(val, pdata->regs + _reg); \
276 res = count; \
277 } \
278 \
279 return res; \
280 } \
281 \
282 DEBUGFS_STRUCT(_name, 0600, \
283 thunderx_##_type##_##_name##_write, \
284 thunderx_##_type##_##_name##_read)
285
286 #define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field)
287
288 /*
289 * To get an ECC error injected, the following steps are needed:
290 * - Setup the ECC injection by writing the appropriate parameters:
291 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
292 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
293 * echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
294 * - Do the actual injection:
295 * echo 1 > /sys/kernel/debug/<device number>/inject_ecc
296 */
thunderx_lmc_inject_int_write(struct file * file,const char __user * data,size_t count,loff_t * ppos)297 static ssize_t thunderx_lmc_inject_int_write(struct file *file,
298 const char __user *data,
299 size_t count, loff_t *ppos)
300 {
301 struct thunderx_lmc *lmc = file->private_data;
302 u64 val;
303 int res;
304
305 res = kstrtoull_from_user(data, count, 0, &val);
306
307 if (!res) {
308 /* Trigger the interrupt */
309 writeq(val, lmc->regs + LMC_INT_W1S);
310 res = count;
311 }
312
313 return res;
314 }
315
thunderx_lmc_int_read(struct file * file,char __user * data,size_t count,loff_t * ppos)316 static ssize_t thunderx_lmc_int_read(struct file *file,
317 char __user *data,
318 size_t count, loff_t *ppos)
319 {
320 struct thunderx_lmc *lmc = file->private_data;
321 char buf[20];
322 u64 lmc_int = readq(lmc->regs + LMC_INT);
323
324 snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
325 return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
326 }
327
328 #define TEST_PATTERN 0xa5
329
inject_ecc_fn(void * arg)330 static int inject_ecc_fn(void *arg)
331 {
332 struct thunderx_lmc *lmc = arg;
333 uintptr_t addr, phys;
334 unsigned int cline_size = cache_line_size();
335 const unsigned int lines = PAGE_SIZE / cline_size;
336 unsigned int i, cl_idx;
337
338 addr = (uintptr_t)page_address(lmc->mem);
339 phys = (uintptr_t)page_to_phys(lmc->mem);
340
341 cl_idx = (phys & 0x7f) >> 4;
342 lmc->parity_test &= ~(7ULL << 8);
343 lmc->parity_test |= (cl_idx << 8);
344
345 writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
346 writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
347 writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
348
349 readq(lmc->regs + LMC_CHAR_MASK0);
350 readq(lmc->regs + LMC_CHAR_MASK2);
351 readq(lmc->regs + LMC_ECC_PARITY_TEST);
352
353 for (i = 0; i < lines; i++) {
354 memset((void *)addr, TEST_PATTERN, cline_size);
355 barrier();
356
357 /*
358 * Flush L1 cachelines to the PoC (L2).
359 * This will cause cacheline eviction to the L2.
360 */
361 asm volatile("dc civac, %0\n"
362 "dsb sy\n"
363 : : "r"(addr + i * cline_size));
364 }
365
366 for (i = 0; i < lines; i++) {
367 /*
368 * Flush L2 cachelines to the DRAM.
369 * This will cause cacheline eviction to the DRAM
370 * and ECC corruption according to the masks set.
371 */
372 __asm__ volatile("sys #0,c11,C1,#2, %0\n"
373 : : "r"(phys + i * cline_size));
374 }
375
376 for (i = 0; i < lines; i++) {
377 /*
378 * Invalidate L2 cachelines.
379 * The subsequent load will cause cacheline fetch
380 * from the DRAM and an error interrupt
381 */
382 __asm__ volatile("sys #0,c11,C1,#1, %0"
383 : : "r"(phys + i * cline_size));
384 }
385
386 for (i = 0; i < lines; i++) {
387 /*
388 * Invalidate L1 cachelines.
389 * The subsequent load will cause cacheline fetch
390 * from the L2 and/or DRAM
391 */
392 asm volatile("dc ivac, %0\n"
393 "dsb sy\n"
394 : : "r"(addr + i * cline_size));
395 }
396
397 return 0;
398 }
399
thunderx_lmc_inject_ecc_write(struct file * file,const char __user * data,size_t count,loff_t * ppos)400 static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
401 const char __user *data,
402 size_t count, loff_t *ppos)
403 {
404 struct thunderx_lmc *lmc = file->private_data;
405 unsigned int cline_size = cache_line_size();
406 u8 *tmp;
407 void __iomem *addr;
408 unsigned int offs, timeout = 100000;
409
410 atomic_set(&lmc->ecc_int, 0);
411
412 lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
413 if (!lmc->mem)
414 return -ENOMEM;
415
416 tmp = kmalloc(cline_size, GFP_KERNEL);
417 if (!tmp) {
418 __free_pages(lmc->mem, 0);
419 return -ENOMEM;
420 }
421
422 addr = page_address(lmc->mem);
423
424 while (!atomic_read(&lmc->ecc_int) && timeout--) {
425 stop_machine(inject_ecc_fn, lmc, NULL);
426
427 for (offs = 0; offs < PAGE_SIZE; offs += cline_size) {
428 /*
429 * Do a load from the previously rigged location
430 * This should generate an error interrupt.
431 */
432 memcpy(tmp, addr + offs, cline_size);
433 asm volatile("dsb ld\n");
434 }
435 }
436
437 kfree(tmp);
438 __free_pages(lmc->mem, 0);
439
440 return count;
441 }
442
443 LMC_DEBUGFS_ENT(mask0);
444 LMC_DEBUGFS_ENT(mask2);
445 LMC_DEBUGFS_ENT(parity_test);
446
447 DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
448 DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
449 DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
450
451 static struct debugfs_entry *lmc_dfs_ents[] = {
452 &debugfs_mask0,
453 &debugfs_mask2,
454 &debugfs_parity_test,
455 &debugfs_inject_ecc,
456 &debugfs_inject_int,
457 &debugfs_int_w1c,
458 };
459
thunderx_create_debugfs_nodes(struct dentry * parent,struct debugfs_entry * attrs[],void * data,size_t num)460 static int thunderx_create_debugfs_nodes(struct dentry *parent,
461 struct debugfs_entry *attrs[],
462 void *data,
463 size_t num)
464 {
465 int i;
466 struct dentry *ent;
467
468 if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
469 return 0;
470
471 if (!parent)
472 return -ENOENT;
473
474 for (i = 0; i < num; i++) {
475 ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
476 parent, data, &attrs[i]->fops);
477
478 if (IS_ERR(ent))
479 break;
480 }
481
482 return i;
483 }
484
thunderx_faddr_to_phys(u64 faddr,struct thunderx_lmc * lmc)485 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
486 {
487 phys_addr_t addr = 0;
488 int bank, xbits;
489
490 addr |= lmc->node << 40;
491 addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
492 addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
493 addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
494 addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
495
496 bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
497
498 if (lmc->xor_bank)
499 bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
500
501 addr |= bank << lmc->bank_lsb;
502
503 xbits = PCI_FUNC(lmc->pdev->devfn);
504
505 if (lmc->l2c_alias)
506 xbits ^= get_bits(addr, 20, lmc->xbits) ^
507 get_bits(addr, 12, lmc->xbits);
508
509 addr |= xbits << 7;
510
511 return addr;
512 }
513
thunderx_get_num_lmcs(unsigned int node)514 static unsigned int thunderx_get_num_lmcs(unsigned int node)
515 {
516 unsigned int number = 0;
517 struct pci_dev *pdev = NULL;
518
519 do {
520 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
521 PCI_DEVICE_ID_THUNDER_LMC,
522 pdev);
523 if (pdev) {
524 #ifdef CONFIG_NUMA
525 if (pdev->dev.numa_node == node)
526 number++;
527 #else
528 number++;
529 #endif
530 }
531 } while (pdev);
532
533 return number;
534 }
535
536 #define LMC_MESSAGE_SIZE 120
537 #define LMC_OTHER_SIZE (50 * ARRAY_SIZE(lmc_errors))
538
thunderx_lmc_err_isr(int irq,void * dev_id)539 static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
540 {
541 struct mem_ctl_info *mci = dev_id;
542 struct thunderx_lmc *lmc = mci->pvt_info;
543
544 unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
545 struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
546
547 writeq(0, lmc->regs + LMC_CHAR_MASK0);
548 writeq(0, lmc->regs + LMC_CHAR_MASK2);
549 writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
550
551 ctx->reg_int = readq(lmc->regs + LMC_INT);
552 ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
553 ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
554 ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
555 ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
556
557 lmc->ring_head++;
558
559 atomic_set(&lmc->ecc_int, 1);
560
561 /* Clear the interrupt */
562 writeq(ctx->reg_int, lmc->regs + LMC_INT);
563
564 return IRQ_WAKE_THREAD;
565 }
566
thunderx_lmc_threaded_isr(int irq,void * dev_id)567 static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
568 {
569 struct mem_ctl_info *mci = dev_id;
570 struct thunderx_lmc *lmc = mci->pvt_info;
571 phys_addr_t phys_addr;
572
573 unsigned long tail;
574 struct lmc_err_ctx *ctx;
575
576 irqreturn_t ret = IRQ_NONE;
577
578 char *msg;
579 char *other;
580
581 msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
582 other = kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
583
584 if (!msg || !other)
585 goto err_free;
586
587 while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
588 ARRAY_SIZE(lmc->err_ctx))) {
589 tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
590
591 ctx = &lmc->err_ctx[tail];
592
593 dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
594 ctx->reg_int);
595 dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
596 ctx->reg_fadr);
597 dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
598 ctx->reg_nxm_fadr);
599 dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
600 ctx->reg_scram_fadr);
601 dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
602 ctx->reg_ecc_synd);
603
604 snprintf(msg, LMC_MESSAGE_SIZE,
605 "DIMM %lld rank %lld bank %lld row %lld col %lld",
606 LMC_FADR_FDIMM(ctx->reg_scram_fadr),
607 LMC_FADR_FBUNK(ctx->reg_scram_fadr),
608 LMC_FADR_FBANK(ctx->reg_scram_fadr),
609 LMC_FADR_FROW(ctx->reg_scram_fadr),
610 LMC_FADR_FCOL(ctx->reg_scram_fadr));
611
612 decode_register(other, LMC_OTHER_SIZE, lmc_errors,
613 ctx->reg_int);
614
615 phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
616
617 if (ctx->reg_int & LMC_INT_UE)
618 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
619 phys_to_pfn(phys_addr),
620 offset_in_page(phys_addr),
621 0, -1, -1, -1, msg, other);
622 else if (ctx->reg_int & LMC_INT_CE)
623 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
624 phys_to_pfn(phys_addr),
625 offset_in_page(phys_addr),
626 0, -1, -1, -1, msg, other);
627
628 lmc->ring_tail++;
629 }
630
631 ret = IRQ_HANDLED;
632
633 err_free:
634 kfree(msg);
635 kfree(other);
636
637 return ret;
638 }
639
640 static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
641 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
642 { 0, },
643 };
644
pci_dev_to_mc_idx(struct pci_dev * pdev)645 static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
646 {
647 int node = dev_to_node(&pdev->dev);
648 int ret = PCI_FUNC(pdev->devfn);
649
650 ret += max(node, 0) << 3;
651
652 return ret;
653 }
654
thunderx_lmc_probe(struct pci_dev * pdev,const struct pci_device_id * id)655 static int thunderx_lmc_probe(struct pci_dev *pdev,
656 const struct pci_device_id *id)
657 {
658 struct thunderx_lmc *lmc;
659 struct edac_mc_layer layer;
660 struct mem_ctl_info *mci;
661 u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
662 int ret;
663 u64 lmc_int;
664 void *l2c_ioaddr;
665
666 layer.type = EDAC_MC_LAYER_SLOT;
667 layer.size = 2;
668 layer.is_virt_csrow = false;
669
670 ret = pcim_enable_device(pdev);
671 if (ret) {
672 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
673 return ret;
674 }
675
676 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
677 if (ret) {
678 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
679 return ret;
680 }
681
682 mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
683 sizeof(struct thunderx_lmc));
684 if (!mci)
685 return -ENOMEM;
686
687 mci->pdev = &pdev->dev;
688 lmc = mci->pvt_info;
689
690 pci_set_drvdata(pdev, mci);
691
692 lmc->regs = pcim_iomap_table(pdev)[0];
693
694 lmc_control = readq(lmc->regs + LMC_CONTROL);
695 lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
696 lmc_config = readq(lmc->regs + LMC_CONFIG);
697
698 if (lmc_control & LMC_CONTROL_RDIMM) {
699 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
700 lmc_ddr_pll_ctl) ?
701 MEM_RDDR4 : MEM_RDDR3;
702 } else {
703 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
704 lmc_ddr_pll_ctl) ?
705 MEM_DDR4 : MEM_DDR3;
706 }
707
708 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
709 mci->edac_cap = EDAC_FLAG_SECDED;
710
711 mci->mod_name = "thunderx-lmc";
712 mci->ctl_name = "thunderx-lmc";
713 mci->dev_name = dev_name(&pdev->dev);
714 mci->scrub_mode = SCRUB_NONE;
715
716 lmc->pdev = pdev;
717 lmc->msix_ent.entry = 0;
718
719 lmc->ring_head = 0;
720 lmc->ring_tail = 0;
721
722 ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
723 if (ret) {
724 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
725 goto err_free;
726 }
727
728 ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
729 thunderx_lmc_err_isr,
730 thunderx_lmc_threaded_isr, 0,
731 "[EDAC] ThunderX LMC", mci);
732 if (ret)
733 goto err_free;
734
735 lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
736
737 lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
738 lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
739 FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
740
741 lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
742 lmc->dimm_lsb = 28 + lmc->pbank_lsb + lmc->xbits;
743 lmc->rank_lsb = lmc->dimm_lsb;
744 lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
745 lmc->bank_lsb = 7 + lmc->xbits;
746 lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
747
748 lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
749
750 lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
751
752 l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
753 if (!l2c_ioaddr) {
754 dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
755 ret = -ENOMEM;
756 goto err_free;
757 }
758
759 lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
760
761 iounmap(l2c_ioaddr);
762
763 ret = edac_mc_add_mc(mci);
764 if (ret) {
765 dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
766 goto err_free;
767 }
768
769 lmc_int = readq(lmc->regs + LMC_INT);
770 writeq(lmc_int, lmc->regs + LMC_INT);
771
772 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
773
774 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
775 ret = thunderx_create_debugfs_nodes(mci->debugfs,
776 lmc_dfs_ents,
777 lmc,
778 ARRAY_SIZE(lmc_dfs_ents));
779
780 if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
781 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
782 ret, ret >= 0 ? " created" : "");
783 }
784 }
785
786 return 0;
787
788 err_free:
789 pci_set_drvdata(pdev, NULL);
790 edac_mc_free(mci);
791
792 return ret;
793 }
794
thunderx_lmc_remove(struct pci_dev * pdev)795 static void thunderx_lmc_remove(struct pci_dev *pdev)
796 {
797 struct mem_ctl_info *mci = pci_get_drvdata(pdev);
798 struct thunderx_lmc *lmc = mci->pvt_info;
799
800 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
801
802 edac_mc_del_mc(&pdev->dev);
803 edac_mc_free(mci);
804 }
805
806 MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
807
808 static struct pci_driver thunderx_lmc_driver = {
809 .name = "thunderx_lmc_edac",
810 .probe = thunderx_lmc_probe,
811 .remove = thunderx_lmc_remove,
812 .id_table = thunderx_lmc_pci_tbl,
813 };
814
815 /*---------------------- OCX driver ---------------------------------*/
816
817 #define PCI_DEVICE_ID_THUNDER_OCX 0xa013
818
819 #define OCX_LINK_INTS 3
820 #define OCX_INTS (OCX_LINK_INTS + 1)
821 #define OCX_RX_LANES 24
822 #define OCX_RX_LANE_STATS 15
823
824 #define OCX_COM_INT 0x100
825 #define OCX_COM_INT_W1S 0x108
826 #define OCX_COM_INT_ENA_W1S 0x110
827 #define OCX_COM_INT_ENA_W1C 0x118
828
829 #define OCX_COM_IO_BADID BIT(54)
830 #define OCX_COM_MEM_BADID BIT(53)
831 #define OCX_COM_COPR_BADID BIT(52)
832 #define OCX_COM_WIN_REQ_BADID BIT(51)
833 #define OCX_COM_WIN_REQ_TOUT BIT(50)
834 #define OCX_COM_RX_LANE GENMASK(23, 0)
835
836 #define OCX_COM_INT_CE (OCX_COM_IO_BADID | \
837 OCX_COM_MEM_BADID | \
838 OCX_COM_COPR_BADID | \
839 OCX_COM_WIN_REQ_BADID | \
840 OCX_COM_WIN_REQ_TOUT)
841
842 static const struct error_descr ocx_com_errors[] = {
843 {
844 .type = ERR_CORRECTED,
845 .mask = OCX_COM_IO_BADID,
846 .descr = "Invalid IO transaction node ID",
847 },
848 {
849 .type = ERR_CORRECTED,
850 .mask = OCX_COM_MEM_BADID,
851 .descr = "Invalid memory transaction node ID",
852 },
853 {
854 .type = ERR_CORRECTED,
855 .mask = OCX_COM_COPR_BADID,
856 .descr = "Invalid coprocessor transaction node ID",
857 },
858 {
859 .type = ERR_CORRECTED,
860 .mask = OCX_COM_WIN_REQ_BADID,
861 .descr = "Invalid SLI transaction node ID",
862 },
863 {
864 .type = ERR_CORRECTED,
865 .mask = OCX_COM_WIN_REQ_TOUT,
866 .descr = "Window/core request timeout",
867 },
868 {0, 0, NULL},
869 };
870
871 #define OCX_COM_LINKX_INT(x) (0x120 + (x) * 8)
872 #define OCX_COM_LINKX_INT_W1S(x) (0x140 + (x) * 8)
873 #define OCX_COM_LINKX_INT_ENA_W1S(x) (0x160 + (x) * 8)
874 #define OCX_COM_LINKX_INT_ENA_W1C(x) (0x180 + (x) * 8)
875
876 #define OCX_COM_LINK_BAD_WORD BIT(13)
877 #define OCX_COM_LINK_ALIGN_FAIL BIT(12)
878 #define OCX_COM_LINK_ALIGN_DONE BIT(11)
879 #define OCX_COM_LINK_UP BIT(10)
880 #define OCX_COM_LINK_STOP BIT(9)
881 #define OCX_COM_LINK_BLK_ERR BIT(8)
882 #define OCX_COM_LINK_REINIT BIT(7)
883 #define OCX_COM_LINK_LNK_DATA BIT(6)
884 #define OCX_COM_LINK_RXFIFO_DBE BIT(5)
885 #define OCX_COM_LINK_RXFIFO_SBE BIT(4)
886 #define OCX_COM_LINK_TXFIFO_DBE BIT(3)
887 #define OCX_COM_LINK_TXFIFO_SBE BIT(2)
888 #define OCX_COM_LINK_REPLAY_DBE BIT(1)
889 #define OCX_COM_LINK_REPLAY_SBE BIT(0)
890
891 static const struct error_descr ocx_com_link_errors[] = {
892 {
893 .type = ERR_CORRECTED,
894 .mask = OCX_COM_LINK_REPLAY_SBE,
895 .descr = "Replay buffer single-bit error",
896 },
897 {
898 .type = ERR_CORRECTED,
899 .mask = OCX_COM_LINK_TXFIFO_SBE,
900 .descr = "TX FIFO single-bit error",
901 },
902 {
903 .type = ERR_CORRECTED,
904 .mask = OCX_COM_LINK_RXFIFO_SBE,
905 .descr = "RX FIFO single-bit error",
906 },
907 {
908 .type = ERR_CORRECTED,
909 .mask = OCX_COM_LINK_BLK_ERR,
910 .descr = "Block code error",
911 },
912 {
913 .type = ERR_CORRECTED,
914 .mask = OCX_COM_LINK_ALIGN_FAIL,
915 .descr = "Link alignment failure",
916 },
917 {
918 .type = ERR_CORRECTED,
919 .mask = OCX_COM_LINK_BAD_WORD,
920 .descr = "Bad code word",
921 },
922 {
923 .type = ERR_UNCORRECTED,
924 .mask = OCX_COM_LINK_REPLAY_DBE,
925 .descr = "Replay buffer double-bit error",
926 },
927 {
928 .type = ERR_UNCORRECTED,
929 .mask = OCX_COM_LINK_TXFIFO_DBE,
930 .descr = "TX FIFO double-bit error",
931 },
932 {
933 .type = ERR_UNCORRECTED,
934 .mask = OCX_COM_LINK_RXFIFO_DBE,
935 .descr = "RX FIFO double-bit error",
936 },
937 {
938 .type = ERR_UNCORRECTED,
939 .mask = OCX_COM_LINK_STOP,
940 .descr = "Link stopped",
941 },
942 {0, 0, NULL},
943 };
944
945 #define OCX_COM_LINK_INT_UE (OCX_COM_LINK_REPLAY_DBE | \
946 OCX_COM_LINK_TXFIFO_DBE | \
947 OCX_COM_LINK_RXFIFO_DBE | \
948 OCX_COM_LINK_STOP)
949
950 #define OCX_COM_LINK_INT_CE (OCX_COM_LINK_REPLAY_SBE | \
951 OCX_COM_LINK_TXFIFO_SBE | \
952 OCX_COM_LINK_RXFIFO_SBE | \
953 OCX_COM_LINK_BLK_ERR | \
954 OCX_COM_LINK_ALIGN_FAIL | \
955 OCX_COM_LINK_BAD_WORD)
956
957 #define OCX_LNE_INT(x) (0x8018 + (x) * 0x100)
958 #define OCX_LNE_INT_EN(x) (0x8020 + (x) * 0x100)
959 #define OCX_LNE_BAD_CNT(x) (0x8028 + (x) * 0x100)
960 #define OCX_LNE_CFG(x) (0x8000 + (x) * 0x100)
961 #define OCX_LNE_STAT(x, y) (0x8040 + (x) * 0x100 + (y) * 8)
962
963 #define OCX_LNE_CFG_RX_BDRY_LOCK_DIS BIT(8)
964 #define OCX_LNE_CFG_RX_STAT_WRAP_DIS BIT(2)
965 #define OCX_LNE_CFG_RX_STAT_RDCLR BIT(1)
966 #define OCX_LNE_CFG_RX_STAT_ENA BIT(0)
967
968
969 #define OCX_LANE_BAD_64B67B BIT(8)
970 #define OCX_LANE_DSKEW_FIFO_OVFL BIT(5)
971 #define OCX_LANE_SCRM_SYNC_LOSS BIT(4)
972 #define OCX_LANE_UKWN_CNTL_WORD BIT(3)
973 #define OCX_LANE_CRC32_ERR BIT(2)
974 #define OCX_LANE_BDRY_SYNC_LOSS BIT(1)
975 #define OCX_LANE_SERDES_LOCK_LOSS BIT(0)
976
977 #define OCX_COM_LANE_INT_UE (0)
978 #define OCX_COM_LANE_INT_CE (OCX_LANE_SERDES_LOCK_LOSS | \
979 OCX_LANE_BDRY_SYNC_LOSS | \
980 OCX_LANE_CRC32_ERR | \
981 OCX_LANE_UKWN_CNTL_WORD | \
982 OCX_LANE_SCRM_SYNC_LOSS | \
983 OCX_LANE_DSKEW_FIFO_OVFL | \
984 OCX_LANE_BAD_64B67B)
985
986 static const struct error_descr ocx_lane_errors[] = {
987 {
988 .type = ERR_CORRECTED,
989 .mask = OCX_LANE_SERDES_LOCK_LOSS,
990 .descr = "RX SerDes lock lost",
991 },
992 {
993 .type = ERR_CORRECTED,
994 .mask = OCX_LANE_BDRY_SYNC_LOSS,
995 .descr = "RX word boundary lost",
996 },
997 {
998 .type = ERR_CORRECTED,
999 .mask = OCX_LANE_CRC32_ERR,
1000 .descr = "CRC32 error",
1001 },
1002 {
1003 .type = ERR_CORRECTED,
1004 .mask = OCX_LANE_UKWN_CNTL_WORD,
1005 .descr = "Unknown control word",
1006 },
1007 {
1008 .type = ERR_CORRECTED,
1009 .mask = OCX_LANE_SCRM_SYNC_LOSS,
1010 .descr = "Scrambler synchronization lost",
1011 },
1012 {
1013 .type = ERR_CORRECTED,
1014 .mask = OCX_LANE_DSKEW_FIFO_OVFL,
1015 .descr = "RX deskew FIFO overflow",
1016 },
1017 {
1018 .type = ERR_CORRECTED,
1019 .mask = OCX_LANE_BAD_64B67B,
1020 .descr = "Bad 64B/67B codeword",
1021 },
1022 {0, 0, NULL},
1023 };
1024
1025 #define OCX_LNE_INT_ENA_ALL (GENMASK(9, 8) | GENMASK(6, 0))
1026 #define OCX_COM_INT_ENA_ALL (GENMASK(54, 50) | GENMASK(23, 0))
1027 #define OCX_COM_LINKX_INT_ENA_ALL (GENMASK(13, 12) | \
1028 GENMASK(9, 7) | GENMASK(5, 0))
1029
1030 #define OCX_TLKX_ECC_CTL(x) (0x10018 + (x) * 0x2000)
1031 #define OCX_RLKX_ECC_CTL(x) (0x18018 + (x) * 0x2000)
1032
1033 struct ocx_com_err_ctx {
1034 u64 reg_com_int;
1035 u64 reg_lane_int[OCX_RX_LANES];
1036 u64 reg_lane_stat11[OCX_RX_LANES];
1037 };
1038
1039 struct ocx_link_err_ctx {
1040 u64 reg_com_link_int;
1041 int link;
1042 };
1043
1044 struct thunderx_ocx {
1045 void __iomem *regs;
1046 int com_link;
1047 struct pci_dev *pdev;
1048 struct edac_device_ctl_info *edac_dev;
1049
1050 struct dentry *debugfs;
1051 struct msix_entry msix_ent[OCX_INTS];
1052
1053 struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1054 struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1055
1056 unsigned long com_ring_head;
1057 unsigned long com_ring_tail;
1058
1059 unsigned long link_ring_head;
1060 unsigned long link_ring_tail;
1061 };
1062
1063 #define OCX_MESSAGE_SIZE SZ_1K
1064 #define OCX_OTHER_SIZE (50 * ARRAY_SIZE(ocx_com_link_errors))
1065
1066 /* This handler is threaded */
thunderx_ocx_com_isr(int irq,void * irq_id)1067 static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1068 {
1069 struct msix_entry *msix = irq_id;
1070 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1071 msix_ent[msix->entry]);
1072
1073 int lane;
1074 unsigned long head = ring_pos(ocx->com_ring_head,
1075 ARRAY_SIZE(ocx->com_err_ctx));
1076 struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1077
1078 ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1079
1080 for (lane = 0; lane < OCX_RX_LANES; lane++) {
1081 ctx->reg_lane_int[lane] =
1082 readq(ocx->regs + OCX_LNE_INT(lane));
1083 ctx->reg_lane_stat11[lane] =
1084 readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1085
1086 writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1087 }
1088
1089 writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1090
1091 ocx->com_ring_head++;
1092
1093 return IRQ_WAKE_THREAD;
1094 }
1095
thunderx_ocx_com_threaded_isr(int irq,void * irq_id)1096 static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1097 {
1098 struct msix_entry *msix = irq_id;
1099 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1100 msix_ent[msix->entry]);
1101
1102 irqreturn_t ret = IRQ_NONE;
1103
1104 unsigned long tail;
1105 struct ocx_com_err_ctx *ctx;
1106 int lane;
1107 char *msg;
1108 char *other;
1109
1110 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1111 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1112
1113 if (!msg || !other)
1114 goto err_free;
1115
1116 while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1117 ARRAY_SIZE(ocx->com_err_ctx))) {
1118 tail = ring_pos(ocx->com_ring_tail,
1119 ARRAY_SIZE(ocx->com_err_ctx));
1120 ctx = &ocx->com_err_ctx[tail];
1121
1122 snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1123 ocx->edac_dev->ctl_name, ctx->reg_com_int);
1124
1125 decode_register(other, OCX_OTHER_SIZE,
1126 ocx_com_errors, ctx->reg_com_int);
1127
1128 strlcat(msg, other, OCX_MESSAGE_SIZE);
1129
1130 for (lane = 0; lane < OCX_RX_LANES; lane++)
1131 if (ctx->reg_com_int & BIT(lane)) {
1132 snprintf(other, OCX_OTHER_SIZE,
1133 "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1134 lane, ctx->reg_lane_int[lane],
1135 lane, ctx->reg_lane_stat11[lane]);
1136
1137 strlcat(msg, other, OCX_MESSAGE_SIZE);
1138
1139 decode_register(other, OCX_OTHER_SIZE,
1140 ocx_lane_errors,
1141 ctx->reg_lane_int[lane]);
1142 strlcat(msg, other, OCX_MESSAGE_SIZE);
1143 }
1144
1145 if (ctx->reg_com_int & OCX_COM_INT_CE)
1146 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1147
1148 ocx->com_ring_tail++;
1149 }
1150
1151 ret = IRQ_HANDLED;
1152
1153 err_free:
1154 kfree(other);
1155 kfree(msg);
1156
1157 return ret;
1158 }
1159
thunderx_ocx_lnk_isr(int irq,void * irq_id)1160 static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1161 {
1162 struct msix_entry *msix = irq_id;
1163 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1164 msix_ent[msix->entry]);
1165 unsigned long head = ring_pos(ocx->link_ring_head,
1166 ARRAY_SIZE(ocx->link_err_ctx));
1167 struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1168
1169 ctx->link = msix->entry;
1170 ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1171
1172 writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1173
1174 ocx->link_ring_head++;
1175
1176 return IRQ_WAKE_THREAD;
1177 }
1178
thunderx_ocx_lnk_threaded_isr(int irq,void * irq_id)1179 static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1180 {
1181 struct msix_entry *msix = irq_id;
1182 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1183 msix_ent[msix->entry]);
1184 irqreturn_t ret = IRQ_NONE;
1185 unsigned long tail;
1186 struct ocx_link_err_ctx *ctx;
1187
1188 char *msg;
1189 char *other;
1190
1191 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1192 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1193
1194 if (!msg || !other)
1195 goto err_free;
1196
1197 while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1198 ARRAY_SIZE(ocx->link_err_ctx))) {
1199 tail = ring_pos(ocx->link_ring_head,
1200 ARRAY_SIZE(ocx->link_err_ctx));
1201
1202 ctx = &ocx->link_err_ctx[tail];
1203
1204 snprintf(msg, OCX_MESSAGE_SIZE,
1205 "%s: OCX_COM_LINK_INT[%d]: %016llx",
1206 ocx->edac_dev->ctl_name,
1207 ctx->link, ctx->reg_com_link_int);
1208
1209 decode_register(other, OCX_OTHER_SIZE,
1210 ocx_com_link_errors, ctx->reg_com_link_int);
1211
1212 strlcat(msg, other, OCX_MESSAGE_SIZE);
1213
1214 if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1215 edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1216 else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1217 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1218
1219 ocx->link_ring_tail++;
1220 }
1221
1222 ret = IRQ_HANDLED;
1223 err_free:
1224 kfree(other);
1225 kfree(msg);
1226
1227 return ret;
1228 }
1229
1230 #define OCX_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(ocx, _name, _reg)
1231
1232 OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1233 OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1234 OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1235
1236 OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1237 OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1238 OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1239
1240 OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1241 OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1242 OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1243
1244 OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1245 OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1246 OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1247 OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1248 OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1249 OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1250 OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1251 OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1252
1253 OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1254 OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1255 OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1256 OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1257 OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1258 OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1259 OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1260 OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1261
1262 OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1263 OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1264 OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1265 OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1266 OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1267 OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1268 OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1269 OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1270
1271 OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1272
1273 static struct debugfs_entry *ocx_dfs_ents[] = {
1274 &debugfs_tlk0_ecc_ctl,
1275 &debugfs_tlk1_ecc_ctl,
1276 &debugfs_tlk2_ecc_ctl,
1277
1278 &debugfs_rlk0_ecc_ctl,
1279 &debugfs_rlk1_ecc_ctl,
1280 &debugfs_rlk2_ecc_ctl,
1281
1282 &debugfs_com_link0_int,
1283 &debugfs_com_link1_int,
1284 &debugfs_com_link2_int,
1285
1286 &debugfs_lne00_badcnt,
1287 &debugfs_lne01_badcnt,
1288 &debugfs_lne02_badcnt,
1289 &debugfs_lne03_badcnt,
1290 &debugfs_lne04_badcnt,
1291 &debugfs_lne05_badcnt,
1292 &debugfs_lne06_badcnt,
1293 &debugfs_lne07_badcnt,
1294 &debugfs_lne08_badcnt,
1295 &debugfs_lne09_badcnt,
1296 &debugfs_lne10_badcnt,
1297 &debugfs_lne11_badcnt,
1298 &debugfs_lne12_badcnt,
1299 &debugfs_lne13_badcnt,
1300 &debugfs_lne14_badcnt,
1301 &debugfs_lne15_badcnt,
1302 &debugfs_lne16_badcnt,
1303 &debugfs_lne17_badcnt,
1304 &debugfs_lne18_badcnt,
1305 &debugfs_lne19_badcnt,
1306 &debugfs_lne20_badcnt,
1307 &debugfs_lne21_badcnt,
1308 &debugfs_lne22_badcnt,
1309 &debugfs_lne23_badcnt,
1310
1311 &debugfs_com_int,
1312 };
1313
1314 static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1315 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1316 { 0, },
1317 };
1318
thunderx_ocx_clearstats(struct thunderx_ocx * ocx)1319 static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1320 {
1321 int lane, stat, cfg;
1322
1323 for (lane = 0; lane < OCX_RX_LANES; lane++) {
1324 cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1325 cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1326 cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1327 writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1328
1329 for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1330 readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1331 }
1332 }
1333
thunderx_ocx_probe(struct pci_dev * pdev,const struct pci_device_id * id)1334 static int thunderx_ocx_probe(struct pci_dev *pdev,
1335 const struct pci_device_id *id)
1336 {
1337 struct thunderx_ocx *ocx;
1338 struct edac_device_ctl_info *edac_dev;
1339 char name[32];
1340 int idx;
1341 int i;
1342 int ret;
1343 u64 reg;
1344
1345 ret = pcim_enable_device(pdev);
1346 if (ret) {
1347 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1348 return ret;
1349 }
1350
1351 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1352 if (ret) {
1353 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1354 return ret;
1355 }
1356
1357 idx = edac_device_alloc_index();
1358 snprintf(name, sizeof(name), "OCX%d", idx);
1359 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1360 name, 1, "CCPI", 1, 0, idx);
1361 if (!edac_dev) {
1362 dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
1363 return -ENOMEM;
1364 }
1365 ocx = edac_dev->pvt_info;
1366 ocx->edac_dev = edac_dev;
1367 ocx->com_ring_head = 0;
1368 ocx->com_ring_tail = 0;
1369 ocx->link_ring_head = 0;
1370 ocx->link_ring_tail = 0;
1371
1372 ocx->regs = pcim_iomap_table(pdev)[0];
1373 if (!ocx->regs) {
1374 dev_err(&pdev->dev, "Cannot map PCI resources\n");
1375 ret = -ENODEV;
1376 goto err_free;
1377 }
1378
1379 ocx->pdev = pdev;
1380
1381 for (i = 0; i < OCX_INTS; i++) {
1382 ocx->msix_ent[i].entry = i;
1383 ocx->msix_ent[i].vector = 0;
1384 }
1385
1386 ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1387 if (ret) {
1388 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1389 goto err_free;
1390 }
1391
1392 for (i = 0; i < OCX_INTS; i++) {
1393 ret = devm_request_threaded_irq(&pdev->dev,
1394 ocx->msix_ent[i].vector,
1395 (i == 3) ?
1396 thunderx_ocx_com_isr :
1397 thunderx_ocx_lnk_isr,
1398 (i == 3) ?
1399 thunderx_ocx_com_threaded_isr :
1400 thunderx_ocx_lnk_threaded_isr,
1401 0, "[EDAC] ThunderX OCX",
1402 &ocx->msix_ent[i]);
1403 if (ret)
1404 goto err_free;
1405 }
1406
1407 edac_dev->dev = &pdev->dev;
1408 edac_dev->dev_name = dev_name(&pdev->dev);
1409 edac_dev->mod_name = "thunderx-ocx";
1410 edac_dev->ctl_name = "thunderx-ocx";
1411
1412 ret = edac_device_add_device(edac_dev);
1413 if (ret) {
1414 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1415 goto err_free;
1416 }
1417
1418 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1419 ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1420
1421 ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1422 ocx_dfs_ents,
1423 ocx,
1424 ARRAY_SIZE(ocx_dfs_ents));
1425 if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1426 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1427 ret, ret >= 0 ? " created" : "");
1428 }
1429 }
1430
1431 pci_set_drvdata(pdev, edac_dev);
1432
1433 thunderx_ocx_clearstats(ocx);
1434
1435 for (i = 0; i < OCX_RX_LANES; i++) {
1436 writeq(OCX_LNE_INT_ENA_ALL,
1437 ocx->regs + OCX_LNE_INT_EN(i));
1438
1439 reg = readq(ocx->regs + OCX_LNE_INT(i));
1440 writeq(reg, ocx->regs + OCX_LNE_INT(i));
1441
1442 }
1443
1444 for (i = 0; i < OCX_LINK_INTS; i++) {
1445 reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1446 writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1447
1448 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1449 ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1450 }
1451
1452 reg = readq(ocx->regs + OCX_COM_INT);
1453 writeq(reg, ocx->regs + OCX_COM_INT);
1454
1455 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1456
1457 return 0;
1458 err_free:
1459 edac_device_free_ctl_info(edac_dev);
1460
1461 return ret;
1462 }
1463
thunderx_ocx_remove(struct pci_dev * pdev)1464 static void thunderx_ocx_remove(struct pci_dev *pdev)
1465 {
1466 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1467 struct thunderx_ocx *ocx = edac_dev->pvt_info;
1468 int i;
1469
1470 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1471
1472 for (i = 0; i < OCX_INTS; i++) {
1473 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1474 ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1475 }
1476
1477 edac_debugfs_remove_recursive(ocx->debugfs);
1478
1479 edac_device_del_device(&pdev->dev);
1480 edac_device_free_ctl_info(edac_dev);
1481 }
1482
1483 MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1484
1485 static struct pci_driver thunderx_ocx_driver = {
1486 .name = "thunderx_ocx_edac",
1487 .probe = thunderx_ocx_probe,
1488 .remove = thunderx_ocx_remove,
1489 .id_table = thunderx_ocx_pci_tbl,
1490 };
1491
1492 /*---------------------- L2C driver ---------------------------------*/
1493
1494 #define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1495 #define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1496 #define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1497
1498 #define L2C_TAD_INT_W1C 0x40000
1499 #define L2C_TAD_INT_W1S 0x40008
1500
1501 #define L2C_TAD_INT_ENA_W1C 0x40020
1502 #define L2C_TAD_INT_ENA_W1S 0x40028
1503
1504
1505 #define L2C_TAD_INT_L2DDBE BIT(1)
1506 #define L2C_TAD_INT_SBFSBE BIT(2)
1507 #define L2C_TAD_INT_SBFDBE BIT(3)
1508 #define L2C_TAD_INT_FBFSBE BIT(4)
1509 #define L2C_TAD_INT_FBFDBE BIT(5)
1510 #define L2C_TAD_INT_TAGDBE BIT(9)
1511 #define L2C_TAD_INT_RDDISLMC BIT(15)
1512 #define L2C_TAD_INT_WRDISLMC BIT(16)
1513 #define L2C_TAD_INT_LFBTO BIT(17)
1514 #define L2C_TAD_INT_GSYNCTO BIT(18)
1515 #define L2C_TAD_INT_RTGSBE BIT(32)
1516 #define L2C_TAD_INT_RTGDBE BIT(33)
1517 #define L2C_TAD_INT_RDDISOCI BIT(34)
1518 #define L2C_TAD_INT_WRDISOCI BIT(35)
1519
1520 #define L2C_TAD_INT_ECC (L2C_TAD_INT_L2DDBE | \
1521 L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1522 L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1523
1524 #define L2C_TAD_INT_CE (L2C_TAD_INT_SBFSBE | \
1525 L2C_TAD_INT_FBFSBE)
1526
1527 #define L2C_TAD_INT_UE (L2C_TAD_INT_L2DDBE | \
1528 L2C_TAD_INT_SBFDBE | \
1529 L2C_TAD_INT_FBFDBE | \
1530 L2C_TAD_INT_TAGDBE | \
1531 L2C_TAD_INT_RTGDBE | \
1532 L2C_TAD_INT_WRDISOCI | \
1533 L2C_TAD_INT_RDDISOCI | \
1534 L2C_TAD_INT_WRDISLMC | \
1535 L2C_TAD_INT_RDDISLMC | \
1536 L2C_TAD_INT_LFBTO | \
1537 L2C_TAD_INT_GSYNCTO)
1538
1539 static const struct error_descr l2_tad_errors[] = {
1540 {
1541 .type = ERR_CORRECTED,
1542 .mask = L2C_TAD_INT_SBFSBE,
1543 .descr = "SBF single-bit error",
1544 },
1545 {
1546 .type = ERR_CORRECTED,
1547 .mask = L2C_TAD_INT_FBFSBE,
1548 .descr = "FBF single-bit error",
1549 },
1550 {
1551 .type = ERR_UNCORRECTED,
1552 .mask = L2C_TAD_INT_L2DDBE,
1553 .descr = "L2D double-bit error",
1554 },
1555 {
1556 .type = ERR_UNCORRECTED,
1557 .mask = L2C_TAD_INT_SBFDBE,
1558 .descr = "SBF double-bit error",
1559 },
1560 {
1561 .type = ERR_UNCORRECTED,
1562 .mask = L2C_TAD_INT_FBFDBE,
1563 .descr = "FBF double-bit error",
1564 },
1565 {
1566 .type = ERR_UNCORRECTED,
1567 .mask = L2C_TAD_INT_TAGDBE,
1568 .descr = "TAG double-bit error",
1569 },
1570 {
1571 .type = ERR_UNCORRECTED,
1572 .mask = L2C_TAD_INT_RTGDBE,
1573 .descr = "RTG double-bit error",
1574 },
1575 {
1576 .type = ERR_UNCORRECTED,
1577 .mask = L2C_TAD_INT_WRDISOCI,
1578 .descr = "Write to a disabled CCPI",
1579 },
1580 {
1581 .type = ERR_UNCORRECTED,
1582 .mask = L2C_TAD_INT_RDDISOCI,
1583 .descr = "Read from a disabled CCPI",
1584 },
1585 {
1586 .type = ERR_UNCORRECTED,
1587 .mask = L2C_TAD_INT_WRDISLMC,
1588 .descr = "Write to a disabled LMC",
1589 },
1590 {
1591 .type = ERR_UNCORRECTED,
1592 .mask = L2C_TAD_INT_RDDISLMC,
1593 .descr = "Read from a disabled LMC",
1594 },
1595 {
1596 .type = ERR_UNCORRECTED,
1597 .mask = L2C_TAD_INT_LFBTO,
1598 .descr = "LFB entry timeout",
1599 },
1600 {
1601 .type = ERR_UNCORRECTED,
1602 .mask = L2C_TAD_INT_GSYNCTO,
1603 .descr = "Global sync CCPI timeout",
1604 },
1605 {0, 0, NULL},
1606 };
1607
1608 #define L2C_TAD_INT_TAG (L2C_TAD_INT_TAGDBE)
1609
1610 #define L2C_TAD_INT_RTG (L2C_TAD_INT_RTGDBE)
1611
1612 #define L2C_TAD_INT_DISLMC (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1613
1614 #define L2C_TAD_INT_DISOCI (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1615
1616 #define L2C_TAD_INT_ENA_ALL (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1617 L2C_TAD_INT_RTG | \
1618 L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1619 L2C_TAD_INT_LFBTO)
1620
1621 #define L2C_TAD_TIMETWO 0x50000
1622 #define L2C_TAD_TIMEOUT 0x50100
1623 #define L2C_TAD_ERR 0x60000
1624 #define L2C_TAD_TQD_ERR 0x60100
1625 #define L2C_TAD_TTG_ERR 0x60200
1626
1627
1628 #define L2C_CBC_INT_W1C 0x60000
1629
1630 #define L2C_CBC_INT_RSDSBE BIT(0)
1631 #define L2C_CBC_INT_RSDDBE BIT(1)
1632
1633 #define L2C_CBC_INT_RSD (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1634
1635 #define L2C_CBC_INT_MIBSBE BIT(4)
1636 #define L2C_CBC_INT_MIBDBE BIT(5)
1637
1638 #define L2C_CBC_INT_MIB (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1639
1640 #define L2C_CBC_INT_IORDDISOCI BIT(6)
1641 #define L2C_CBC_INT_IOWRDISOCI BIT(7)
1642
1643 #define L2C_CBC_INT_IODISOCI (L2C_CBC_INT_IORDDISOCI | \
1644 L2C_CBC_INT_IOWRDISOCI)
1645
1646 #define L2C_CBC_INT_CE (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1647 #define L2C_CBC_INT_UE (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1648
1649
1650 static const struct error_descr l2_cbc_errors[] = {
1651 {
1652 .type = ERR_CORRECTED,
1653 .mask = L2C_CBC_INT_RSDSBE,
1654 .descr = "RSD single-bit error",
1655 },
1656 {
1657 .type = ERR_CORRECTED,
1658 .mask = L2C_CBC_INT_MIBSBE,
1659 .descr = "MIB single-bit error",
1660 },
1661 {
1662 .type = ERR_UNCORRECTED,
1663 .mask = L2C_CBC_INT_RSDDBE,
1664 .descr = "RSD double-bit error",
1665 },
1666 {
1667 .type = ERR_UNCORRECTED,
1668 .mask = L2C_CBC_INT_MIBDBE,
1669 .descr = "MIB double-bit error",
1670 },
1671 {
1672 .type = ERR_UNCORRECTED,
1673 .mask = L2C_CBC_INT_IORDDISOCI,
1674 .descr = "Read from a disabled CCPI",
1675 },
1676 {
1677 .type = ERR_UNCORRECTED,
1678 .mask = L2C_CBC_INT_IOWRDISOCI,
1679 .descr = "Write to a disabled CCPI",
1680 },
1681 {0, 0, NULL},
1682 };
1683
1684 #define L2C_CBC_INT_W1S 0x60008
1685 #define L2C_CBC_INT_ENA_W1C 0x60020
1686
1687 #define L2C_CBC_INT_ENA_ALL (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1688 L2C_CBC_INT_IODISOCI)
1689
1690 #define L2C_CBC_INT_ENA_W1S 0x60028
1691
1692 #define L2C_CBC_IODISOCIERR 0x80008
1693 #define L2C_CBC_IOCERR 0x80010
1694 #define L2C_CBC_RSDERR 0x80018
1695 #define L2C_CBC_MIBERR 0x80020
1696
1697
1698 #define L2C_MCI_INT_W1C 0x0
1699
1700 #define L2C_MCI_INT_VBFSBE BIT(0)
1701 #define L2C_MCI_INT_VBFDBE BIT(1)
1702
1703 static const struct error_descr l2_mci_errors[] = {
1704 {
1705 .type = ERR_CORRECTED,
1706 .mask = L2C_MCI_INT_VBFSBE,
1707 .descr = "VBF single-bit error",
1708 },
1709 {
1710 .type = ERR_UNCORRECTED,
1711 .mask = L2C_MCI_INT_VBFDBE,
1712 .descr = "VBF double-bit error",
1713 },
1714 {0, 0, NULL},
1715 };
1716
1717 #define L2C_MCI_INT_W1S 0x8
1718 #define L2C_MCI_INT_ENA_W1C 0x20
1719
1720 #define L2C_MCI_INT_ENA_ALL (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1721
1722 #define L2C_MCI_INT_ENA_W1S 0x28
1723
1724 #define L2C_MCI_ERR 0x10000
1725
1726 #define L2C_MESSAGE_SIZE SZ_1K
1727 #define L2C_OTHER_SIZE (50 * ARRAY_SIZE(l2_tad_errors))
1728
1729 struct l2c_err_ctx {
1730 char *reg_ext_name;
1731 u64 reg_int;
1732 u64 reg_ext;
1733 };
1734
1735 struct thunderx_l2c {
1736 void __iomem *regs;
1737 struct pci_dev *pdev;
1738 struct edac_device_ctl_info *edac_dev;
1739
1740 struct dentry *debugfs;
1741
1742 int index;
1743
1744 struct msix_entry msix_ent;
1745
1746 struct l2c_err_ctx err_ctx[RING_ENTRIES];
1747 unsigned long ring_head;
1748 unsigned long ring_tail;
1749 };
1750
thunderx_l2c_tad_isr(int irq,void * irq_id)1751 static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1752 {
1753 struct msix_entry *msix = irq_id;
1754 struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1755 msix_ent);
1756
1757 unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1758 struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1759
1760 ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1761
1762 if (ctx->reg_int & L2C_TAD_INT_ECC) {
1763 ctx->reg_ext_name = "TQD_ERR";
1764 ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1765 } else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1766 ctx->reg_ext_name = "TTG_ERR";
1767 ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1768 } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1769 ctx->reg_ext_name = "TIMEOUT";
1770 ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1771 } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1772 ctx->reg_ext_name = "ERR";
1773 ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1774 }
1775
1776 writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1777
1778 tad->ring_head++;
1779
1780 return IRQ_WAKE_THREAD;
1781 }
1782
thunderx_l2c_cbc_isr(int irq,void * irq_id)1783 static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1784 {
1785 struct msix_entry *msix = irq_id;
1786 struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1787 msix_ent);
1788
1789 unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1790 struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1791
1792 ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1793
1794 if (ctx->reg_int & L2C_CBC_INT_RSD) {
1795 ctx->reg_ext_name = "RSDERR";
1796 ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1797 } else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1798 ctx->reg_ext_name = "MIBERR";
1799 ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1800 } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1801 ctx->reg_ext_name = "IODISOCIERR";
1802 ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1803 }
1804
1805 writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1806
1807 cbc->ring_head++;
1808
1809 return IRQ_WAKE_THREAD;
1810 }
1811
thunderx_l2c_mci_isr(int irq,void * irq_id)1812 static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1813 {
1814 struct msix_entry *msix = irq_id;
1815 struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1816 msix_ent);
1817
1818 unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1819 struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1820
1821 ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1822 ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1823
1824 writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1825
1826 ctx->reg_ext_name = "ERR";
1827
1828 mci->ring_head++;
1829
1830 return IRQ_WAKE_THREAD;
1831 }
1832
thunderx_l2c_threaded_isr(int irq,void * irq_id)1833 static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1834 {
1835 struct msix_entry *msix = irq_id;
1836 struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1837 msix_ent);
1838
1839 unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1840 struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1841 irqreturn_t ret = IRQ_NONE;
1842
1843 u64 mask_ue, mask_ce;
1844 const struct error_descr *l2_errors;
1845 char *reg_int_name;
1846
1847 char *msg;
1848 char *other;
1849
1850 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1851 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1852
1853 if (!msg || !other)
1854 goto err_free;
1855
1856 switch (l2c->pdev->device) {
1857 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1858 reg_int_name = "L2C_TAD_INT";
1859 mask_ue = L2C_TAD_INT_UE;
1860 mask_ce = L2C_TAD_INT_CE;
1861 l2_errors = l2_tad_errors;
1862 break;
1863 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1864 reg_int_name = "L2C_CBC_INT";
1865 mask_ue = L2C_CBC_INT_UE;
1866 mask_ce = L2C_CBC_INT_CE;
1867 l2_errors = l2_cbc_errors;
1868 break;
1869 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1870 reg_int_name = "L2C_MCI_INT";
1871 mask_ue = L2C_MCI_INT_VBFDBE;
1872 mask_ce = L2C_MCI_INT_VBFSBE;
1873 l2_errors = l2_mci_errors;
1874 break;
1875 default:
1876 dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1877 l2c->pdev->device);
1878 goto err_free;
1879 }
1880
1881 while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1882 ARRAY_SIZE(l2c->err_ctx))) {
1883 snprintf(msg, L2C_MESSAGE_SIZE,
1884 "%s: %s: %016llx, %s: %016llx",
1885 l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1886 ctx->reg_ext_name, ctx->reg_ext);
1887
1888 decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1889
1890 strlcat(msg, other, L2C_MESSAGE_SIZE);
1891
1892 if (ctx->reg_int & mask_ue)
1893 edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1894 else if (ctx->reg_int & mask_ce)
1895 edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1896
1897 l2c->ring_tail++;
1898 }
1899
1900 ret = IRQ_HANDLED;
1901
1902 err_free:
1903 kfree(other);
1904 kfree(msg);
1905
1906 return ret;
1907 }
1908
1909 #define L2C_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(l2c, _name, _reg)
1910
1911 L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1912
1913 static struct debugfs_entry *l2c_tad_dfs_ents[] = {
1914 &debugfs_tad_int,
1915 };
1916
1917 L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1918
1919 static struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1920 &debugfs_cbc_int,
1921 };
1922
1923 L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1924
1925 static struct debugfs_entry *l2c_mci_dfs_ents[] = {
1926 &debugfs_mci_int,
1927 };
1928
1929 static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1930 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1931 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1932 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1933 { 0, },
1934 };
1935
thunderx_l2c_probe(struct pci_dev * pdev,const struct pci_device_id * id)1936 static int thunderx_l2c_probe(struct pci_dev *pdev,
1937 const struct pci_device_id *id)
1938 {
1939 struct thunderx_l2c *l2c;
1940 struct edac_device_ctl_info *edac_dev;
1941 struct debugfs_entry **l2c_devattr;
1942 size_t dfs_entries;
1943 irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1944 char name[32];
1945 const char *fmt;
1946 u64 reg_en_offs, reg_en_mask;
1947 int idx;
1948 int ret;
1949
1950 ret = pcim_enable_device(pdev);
1951 if (ret) {
1952 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1953 return ret;
1954 }
1955
1956 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1957 if (ret) {
1958 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1959 return ret;
1960 }
1961
1962 switch (pdev->device) {
1963 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1964 thunderx_l2c_isr = thunderx_l2c_tad_isr;
1965 l2c_devattr = l2c_tad_dfs_ents;
1966 dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
1967 fmt = "L2C-TAD%d";
1968 reg_en_offs = L2C_TAD_INT_ENA_W1S;
1969 reg_en_mask = L2C_TAD_INT_ENA_ALL;
1970 break;
1971 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1972 thunderx_l2c_isr = thunderx_l2c_cbc_isr;
1973 l2c_devattr = l2c_cbc_dfs_ents;
1974 dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
1975 fmt = "L2C-CBC%d";
1976 reg_en_offs = L2C_CBC_INT_ENA_W1S;
1977 reg_en_mask = L2C_CBC_INT_ENA_ALL;
1978 break;
1979 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1980 thunderx_l2c_isr = thunderx_l2c_mci_isr;
1981 l2c_devattr = l2c_mci_dfs_ents;
1982 dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
1983 fmt = "L2C-MCI%d";
1984 reg_en_offs = L2C_MCI_INT_ENA_W1S;
1985 reg_en_mask = L2C_MCI_INT_ENA_ALL;
1986 break;
1987 default:
1988 //Should never ever get here
1989 dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
1990 pdev->device);
1991 return -EINVAL;
1992 }
1993
1994 idx = edac_device_alloc_index();
1995 snprintf(name, sizeof(name), fmt, idx);
1996
1997 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
1998 name, 1, "L2C", 1, 0, idx);
1999 if (!edac_dev) {
2000 dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2001 return -ENOMEM;
2002 }
2003
2004 l2c = edac_dev->pvt_info;
2005 l2c->edac_dev = edac_dev;
2006
2007 l2c->regs = pcim_iomap_table(pdev)[0];
2008 if (!l2c->regs) {
2009 dev_err(&pdev->dev, "Cannot map PCI resources\n");
2010 ret = -ENODEV;
2011 goto err_free;
2012 }
2013
2014 l2c->pdev = pdev;
2015
2016 l2c->ring_head = 0;
2017 l2c->ring_tail = 0;
2018
2019 l2c->msix_ent.entry = 0;
2020 l2c->msix_ent.vector = 0;
2021
2022 ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2023 if (ret) {
2024 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2025 goto err_free;
2026 }
2027
2028 ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2029 thunderx_l2c_isr,
2030 thunderx_l2c_threaded_isr,
2031 0, "[EDAC] ThunderX L2C",
2032 &l2c->msix_ent);
2033 if (ret)
2034 goto err_free;
2035
2036 edac_dev->dev = &pdev->dev;
2037 edac_dev->dev_name = dev_name(&pdev->dev);
2038 edac_dev->mod_name = "thunderx-l2c";
2039 edac_dev->ctl_name = "thunderx-l2c";
2040
2041 ret = edac_device_add_device(edac_dev);
2042 if (ret) {
2043 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2044 goto err_free;
2045 }
2046
2047 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2048 l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2049
2050 ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2051 l2c, dfs_entries);
2052
2053 if (ret != dfs_entries) {
2054 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2055 ret, ret >= 0 ? " created" : "");
2056 }
2057 }
2058
2059 pci_set_drvdata(pdev, edac_dev);
2060
2061 writeq(reg_en_mask, l2c->regs + reg_en_offs);
2062
2063 return 0;
2064
2065 err_free:
2066 edac_device_free_ctl_info(edac_dev);
2067
2068 return ret;
2069 }
2070
thunderx_l2c_remove(struct pci_dev * pdev)2071 static void thunderx_l2c_remove(struct pci_dev *pdev)
2072 {
2073 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2074 struct thunderx_l2c *l2c = edac_dev->pvt_info;
2075
2076 switch (pdev->device) {
2077 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2078 writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2079 break;
2080 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2081 writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2082 break;
2083 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2084 writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2085 break;
2086 }
2087
2088 edac_debugfs_remove_recursive(l2c->debugfs);
2089
2090 edac_device_del_device(&pdev->dev);
2091 edac_device_free_ctl_info(edac_dev);
2092 }
2093
2094 MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2095
2096 static struct pci_driver thunderx_l2c_driver = {
2097 .name = "thunderx_l2c_edac",
2098 .probe = thunderx_l2c_probe,
2099 .remove = thunderx_l2c_remove,
2100 .id_table = thunderx_l2c_pci_tbl,
2101 };
2102
thunderx_edac_init(void)2103 static int __init thunderx_edac_init(void)
2104 {
2105 int rc = 0;
2106
2107 if (ghes_get_devices())
2108 return -EBUSY;
2109
2110 rc = pci_register_driver(&thunderx_lmc_driver);
2111 if (rc)
2112 return rc;
2113
2114 rc = pci_register_driver(&thunderx_ocx_driver);
2115 if (rc)
2116 goto err_lmc;
2117
2118 rc = pci_register_driver(&thunderx_l2c_driver);
2119 if (rc)
2120 goto err_ocx;
2121
2122 return rc;
2123 err_ocx:
2124 pci_unregister_driver(&thunderx_ocx_driver);
2125 err_lmc:
2126 pci_unregister_driver(&thunderx_lmc_driver);
2127
2128 return rc;
2129 }
2130
thunderx_edac_exit(void)2131 static void __exit thunderx_edac_exit(void)
2132 {
2133 pci_unregister_driver(&thunderx_l2c_driver);
2134 pci_unregister_driver(&thunderx_ocx_driver);
2135 pci_unregister_driver(&thunderx_lmc_driver);
2136
2137 }
2138
2139 module_init(thunderx_edac_init);
2140 module_exit(thunderx_edac_exit);
2141
2142 MODULE_LICENSE("GPL v2");
2143 MODULE_AUTHOR("Cavium, Inc.");
2144 MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");
2145