1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Driver for Intel client SoC with integrated memory controller using IBECC
4 *
5 * Copyright (C) 2020 Intel Corporation
6 *
7 * The In-Band ECC (IBECC) IP provides ECC protection to all or specific
8 * regions of the physical memory space. It's used for memory controllers
9 * that don't support the out-of-band ECC which often needs an additional
10 * storage device to each channel for storing ECC data.
11 */
12
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/irq_work.h>
18 #include <linux/llist.h>
19 #include <linux/genalloc.h>
20 #include <linux/edac.h>
21 #include <linux/bits.h>
22 #include <linux/bitfield.h>
23 #include <linux/io.h>
24 #include <asm/mach_traps.h>
25 #include <asm/nmi.h>
26 #include <asm/mce.h>
27
28 #include "edac_mc.h"
29 #include "edac_module.h"
30
31 #define IGEN6_REVISION "v2.5.1"
32
33 #define EDAC_MOD_STR "igen6_edac"
34 #define IGEN6_NMI_NAME "igen6_ibecc"
35
36 /* Debug macros */
37 #define igen6_printk(level, fmt, arg...) \
38 edac_printk(level, "igen6", fmt, ##arg)
39
40 #define igen6_mc_printk(mci, level, fmt, arg...) \
41 edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg)
42
43 #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
44
45 /* Probing upper bound, not a hardware capability limit. */
46 #define MAX_IMC_TO_PROBE 8
47 #define NUM_CHANNELS 2 /* Max channels */
48 #define NUM_DIMMS 2 /* Max DIMMs per channel */
49
50 #define _4GB BIT_ULL(32)
51
52 /* Size of physical memory */
53 #define TOM_OFFSET 0xa0
54 /* Top of low usable DRAM */
55 #define TOLUD_OFFSET 0xbc
56 /* Capability register C */
57 #define CAPID_C_OFFSET 0xec
58 #define CAPID_C_IBECC BIT(15)
59
60 /* Capability register E */
61 #define CAPID_E_OFFSET 0xf0
62 #define CAPID_E_IBECC BIT(12)
63 #define CAPID_E_IBECC_BIT18 BIT(18)
64
65 /* Error Status */
66 #define ERRSTS_OFFSET 0xc8
67 #define ERRSTS_CE BIT_ULL(6)
68 #define ERRSTS_UE BIT_ULL(7)
69
70 /* Error Command */
71 #define ERRCMD_OFFSET 0xca
72 #define ERRCMD_CE BIT_ULL(6)
73 #define ERRCMD_UE BIT_ULL(7)
74
75 /* IBECC MMIO base address */
76 #define IBECC_BASE (res_cfg->ibecc_base)
77 #define IBECC_ACTIVATE_OFFSET IBECC_BASE
78 #define IBECC_ACTIVATE_EN BIT(0)
79
80 /* IBECC error log */
81 #define ECC_ERROR_LOG_OFFSET (IBECC_BASE + res_cfg->ibecc_error_log_offset)
82 #define ECC_ERROR_LOG_CE BIT_ULL(62)
83 #define ECC_ERROR_LOG_UE BIT_ULL(63)
84 #define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61)
85
86 /* Host MMIO base address */
87 #define MCHBAR_OFFSET 0x48
88 #define MCHBAR_EN BIT_ULL(0)
89 #define MCHBAR_SIZE 0x10000
90
91 /* Parameters for the channel decode stage */
92 #define IMC_BASE (res_cfg->imc_base)
93 #define MAD_INTER_CHANNEL_OFFSET IMC_BASE
94 #define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2)
95 #define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3)
96 #define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4)
97 #define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29)
98
99 /* Parameters for DRAM decode stage */
100 #define MAD_INTRA_CH0_OFFSET (IMC_BASE + 4)
101 #define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0)
102
103 /* DIMM characteristics */
104 #define MAD_DIMM_CH0_OFFSET (IMC_BASE + 0xc)
105 #define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29)
106 #define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8)
107 #define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29)
108 #define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25)
109
110 /* Hash for memory controller selection */
111 #define MAD_MC_HASH_OFFSET (IMC_BASE + 0x1b8)
112 #define MAC_MC_HASH_LSB(v) GET_BITFIELD(v, 1, 3)
113
114 /* Hash for channel selection */
115 #define CHANNEL_HASH_OFFSET (IMC_BASE + 0x24)
116 /* Hash for enhanced channel selection */
117 #define CHANNEL_EHASH_OFFSET (IMC_BASE + 0x28)
118 #define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6)
119 #define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26)
120 #define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28)
121
122 /* Parameters for memory slice decode stage */
123 #define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6)
124 #define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26)
125
126 /*
127 * A slice represents a portion of memory space participating in an
128 * interleave relationship within the memory hierarchy.
129 *
130 * It can represent in different levels such as:
131 *
132 * - a pair of memory controllers
133 * - a memory controller
134 * - a memory channel
135 * - a memory sub-channel / DIMM
136 *
137 * +--------+
138 * | |
139 * | Zone 1 |
140 * | |
141 * +--------+ +--------+
142 * | | | |
143 * | | | |
144 * | Zone 0 | | Zone 0 |
145 * | | | |
146 * | | | |
147 * +--------+ +--------+
148 *
149 * Slice L Slice S
150 *
151 * Memory space is divided into:
152 *
153 * - Zone 0 : Interleaved region
154 * - Zone 1 : Non-interleaved region (upper part of the large slice).
155 */
156 struct slice {
157 /* Slice address. */
158 u64 addr;
159 /* Slice that @addr belongs to. */
160 int id;
161 };
162
163 struct igen6_imc {
164 int mc;
165 struct mem_ctl_info *mci;
166 struct pci_dev *pdev;
167 struct device dev;
168 void __iomem *window;
169 u64 size;
170 u64 ch_s_size;
171 int ch_l_map;
172 u64 dimm_s_size[NUM_CHANNELS];
173 u64 dimm_l_size[NUM_CHANNELS];
174 int dimm_l_map[NUM_CHANNELS];
175 };
176
177 static struct res_config {
178 bool machine_check;
179 /* The number of present memory controllers. */
180 int num_imc;
181 /* Host MMIO configuration */
182 u64 reg_mchbar_mask;
183 /* Top of memory */
184 u64 reg_tom_mask;
185 /* Top of upper usable DRAM */
186 u64 reg_touud_mask;
187 /* IBECC error log */
188 u64 reg_eccerrlog_addr_mask;
189 /* MEMSS_PMA_CR registers. */
190 u32 reg_mem_config_offset;
191 u32 reg_mem_config_ddr_type_mask;
192 u32 reg_mem_config_ibecc_en_mask;
193 u32 reg_capabilities_misc_offset;
194 u32 reg_capabilities_misc_ibecc_dis;
195 /* Memory controller registers. */
196 u32 reg_mad_inter_size_mask[NUM_CHANNELS];
197 u64 reg_mad_inter_size_granularity;
198 u32 reg_mad_intra_rank_mask[NUM_DIMMS];
199 u32 reg_mad_intra_width_mask[NUM_DIMMS];
200 u32 reg_mad_intra_density_mask[NUM_DIMMS];
201 u32 imc_base;
202 u32 cmf_base;
203 u32 cmf_size;
204 u32 ms_hash_offset;
205 u32 ibecc_base;
206 u32 ibecc_error_log_offset;
207 /* Get memory type. */
208 enum mem_type (*get_mem_type)(struct igen6_imc *imc);
209 /* Get DRAM chip type. */
210 enum dev_type (*get_dev_type)(struct igen6_imc *imc, int chan, int dimm_l);
211 /* Set imc->ch_{s_size,l_map}. */
212 void (*set_chan_params)(struct igen6_imc *imc);
213 /* Set imc->dimm_{l_size,s_size,l_map}[chan]. */
214 void (*set_dimm_params)(struct igen6_imc *imc, int chan);
215 bool (*ibecc_available)(struct pci_dev *pdev);
216 /* Convert error address logged in IBECC to system physical address */
217 u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc);
218 /* Convert error address logged in IBECC to integrated memory controller address */
219 u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc);
220 } *res_cfg;
221
222 static struct igen6_pvt {
223 void __iomem *memss_pma_cr;
224 u64 ms_hash;
225 u64 ms_s_size;
226 int ms_l_map;
227 struct igen6_imc imc[];
228 } *igen6_pvt;
229
230 /* The top of low usable DRAM */
231 static u32 igen6_tolud;
232 /* The size of physical memory */
233 static u64 igen6_tom;
234
235 struct decoded_addr {
236 int mc;
237 u64 imc_addr;
238 u64 sys_addr;
239 int channel_idx;
240 u64 channel_addr;
241 int sub_channel_idx;
242 u64 sub_channel_addr;
243 };
244
245 struct ecclog_node {
246 struct llist_node llnode;
247 int mc;
248 u64 ecclog;
249 };
250
251 /*
252 * In the NMI handler, the driver uses the lock-less memory allocator
253 * to allocate memory to store the IBECC error logs and links the logs
254 * to the lock-less list. Delay printk() and the work of error reporting
255 * to EDAC core in a worker.
256 */
257 #define ECCLOG_POOL_SIZE PAGE_SIZE
258 static LLIST_HEAD(ecclog_llist);
259 static struct gen_pool *ecclog_pool;
260 static char ecclog_buf[ECCLOG_POOL_SIZE];
261 static struct irq_work ecclog_irq_work;
262 static struct work_struct ecclog_work;
263
264 /* SoC compute die IDs with IBECC capability. */
265 /* Elkhart Lake */
266 #define DID_EHL_SKU5 0x4514
267 #define DID_EHL_SKU6 0x4528
268 #define DID_EHL_SKU7 0x452a
269 #define DID_EHL_SKU8 0x4516
270 #define DID_EHL_SKU9 0x452c
271 #define DID_EHL_SKU10 0x452e
272 #define DID_EHL_SKU11 0x4532
273 #define DID_EHL_SKU12 0x4518
274 #define DID_EHL_SKU13 0x451a
275 #define DID_EHL_SKU14 0x4534
276 #define DID_EHL_SKU15 0x4536
277
278 /* ICL-NNPI */
279 #define DID_ICL_SKU8 0x4581
280 #define DID_ICL_SKU10 0x4585
281 #define DID_ICL_SKU11 0x4589
282 #define DID_ICL_SKU12 0x458d
283
284 /* Tiger Lake */
285 #define DID_TGL_SKU 0x9a14
286
287 /* Alder Lake */
288 #define DID_ADL_SKU1 0x4601
289 #define DID_ADL_SKU2 0x4602
290 #define DID_ADL_SKU3 0x4621
291 #define DID_ADL_SKU4 0x4641
292
293 /* Alder Lake-N */
294 #define DID_ADL_N_SKU1 0x4614
295 #define DID_ADL_N_SKU2 0x4617
296 #define DID_ADL_N_SKU3 0x461b
297 #define DID_ADL_N_SKU4 0x461c
298 #define DID_ADL_N_SKU5 0x4673
299 #define DID_ADL_N_SKU6 0x4674
300 #define DID_ADL_N_SKU7 0x4675
301 #define DID_ADL_N_SKU8 0x4677
302 #define DID_ADL_N_SKU9 0x4678
303 #define DID_ADL_N_SKU10 0x4679
304 #define DID_ADL_N_SKU11 0x467c
305 #define DID_ADL_N_SKU12 0x4632
306
307 /* Arizona Beach */
308 #define DID_AZB_SKU1 0x4676
309
310 /* Amston Lake */
311 #define DID_ASL_SKU1 0x464a
312 #define DID_ASL_SKU2 0x4646
313 #define DID_ASL_SKU3 0x4652
314
315 /* Raptor Lake-P */
316 #define DID_RPL_P_SKU1 0xa706
317 #define DID_RPL_P_SKU2 0xa707
318 #define DID_RPL_P_SKU3 0xa708
319 #define DID_RPL_P_SKU4 0xa716
320 #define DID_RPL_P_SKU5 0xa718
321
322 /* Meteor Lake-PS */
323 #define DID_MTL_PS_SKU1 0x7d21
324 #define DID_MTL_PS_SKU2 0x7d22
325 #define DID_MTL_PS_SKU3 0x7d23
326 #define DID_MTL_PS_SKU4 0x7d24
327
328 /* Meteor Lake-P */
329 #define DID_MTL_P_SKU1 0x7d01
330 #define DID_MTL_P_SKU2 0x7d02
331 #define DID_MTL_P_SKU3 0x7d14
332
333 /* Arrow Lake-UH */
334 #define DID_ARL_UH_SKU1 0x7d06
335 #define DID_ARL_UH_SKU2 0x7d20
336 #define DID_ARL_UH_SKU3 0x7d30
337
338 /* Panther Lake-H */
339 #define DID_PTL_H_SKU1 0xb000
340 #define DID_PTL_H_SKU2 0xb001
341 #define DID_PTL_H_SKU3 0xb002
342 #define DID_PTL_H_SKU4 0xb003
343 #define DID_PTL_H_SKU5 0xb004
344 #define DID_PTL_H_SKU6 0xb005
345 #define DID_PTL_H_SKU7 0xb008
346 #define DID_PTL_H_SKU8 0xb011
347 #define DID_PTL_H_SKU9 0xb014
348 #define DID_PTL_H_SKU10 0xb015
349 #define DID_PTL_H_SKU11 0xb028
350 #define DID_PTL_H_SKU12 0xb029
351 #define DID_PTL_H_SKU13 0xb02a
352 #define DID_PTL_H_SKU14 0xb00a
353
354 /* Starfire */
355 #define DID_STF_SKU1 0xb02b
356
357 /* Wildcat Lake */
358 #define DID_WCL_SKU1 0xfd00
359
360 /* Nova Lake-H/HX */
361 #define DID_NVL_H_SKU1 0xd701
362 #define DID_NVL_H_SKU2 0xd702
363 #define DID_NVL_H_SKU3 0xd704
364 #define DID_NVL_H_SKU4 0xd705
365
366 /* Remove the interleave bit and shift upper part down to fill gap. */
squeeze_addr(u64 addr,int intlv_bit)367 static u64 squeeze_addr(u64 addr, int intlv_bit)
368 {
369 u64 slice_addr;
370
371 slice_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit;
372 slice_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1);
373
374 return slice_addr;
375 }
376
377 /* Shift the upper bits up and insert a zero at the @intlv_bit bit position. */
inflate_addr(u64 addr,int intlv_bit)378 static u64 inflate_addr(u64 addr, int intlv_bit)
379 {
380 u64 inflated_addr;
381
382 /* Insert a zero at @intlv_bit position. */
383 inflated_addr = GET_BITFIELD(addr, intlv_bit, 63) << (intlv_bit + 1);
384 inflated_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1);
385
386 return inflated_addr;
387 }
388
compute_hash(u64 addr,u64 hash_mask,u64 hash_base,int intlv_bit)389 static u64 compute_hash(u64 addr, u64 hash_mask, u64 hash_base, int intlv_bit)
390 {
391 u64 hash_addr;
392 int i;
393
394 /*
395 * In hash mode, @intlv_bit is the lowest selected bit of @addr
396 * to be XORed. While @mask may or may not include this @intlv_bit,
397 * we enforce that @mask includes @intlv_bit to ensure @intlv_bit is
398 * XORed exactly once.
399 */
400 hash_mask |= BIT_ULL(intlv_bit);
401 hash_addr = addr & hash_mask;
402
403 for (i = 6; i < 20; i++)
404 hash_base ^= (hash_addr >> i) & 1;
405
406 return hash_base;
407 }
408
409 /*
410 * Converts a higher-level address (system / IMC / channel) into a lower-level
411 * slice address and identifier.
412 */
translate_to_lower_level(u64 addr,u64 hash_mask,u64 hash_base,int intlv_bit,u64 s_size,int l_map,struct slice * slice)413 static void translate_to_lower_level(u64 addr, u64 hash_mask, u64 hash_base,
414 int intlv_bit, u64 s_size, int l_map,
415 struct slice *slice)
416 {
417 /* In non-interleave zone. */
418 if (addr >= 2 * s_size) {
419 slice->addr = addr - s_size;
420 slice->id = l_map;
421 return;
422 }
423
424 /* In interleave zone. */
425 slice->addr = squeeze_addr(addr, intlv_bit);
426
427 /* Non-hash mode. */
428 if (!hash_mask) {
429 slice->id = GET_BITFIELD(addr, intlv_bit, intlv_bit);
430 return;
431 }
432
433 /* Hash mode. */
434 slice->id = compute_hash(addr, hash_mask, hash_base, intlv_bit);
435 }
436
437 /* Reconstruct address for upper memory hierarchy level. */
translate_to_upper_level(u64 addr,u64 hash_mask,u64 hash_base,int intlv_bit,u64 s_size)438 static u64 translate_to_upper_level(u64 addr, u64 hash_mask, u64 hash_base,
439 int intlv_bit, u64 s_size)
440 {
441 u64 inflated_addr, hash_val;
442
443 /* In non-interleave zone. */
444 if (addr >= s_size)
445 return addr + s_size;
446
447 /*
448 * In interleave zone.
449 *
450 * Insert a zero at @intlv_bit position.
451 */
452 inflated_addr = inflate_addr(addr, intlv_bit);
453
454 /*
455 * Reconstruct the removed interleave bit and use it to replace
456 * the zero at @intlv_bit position.
457 */
458 hash_val = compute_hash(inflated_addr, hash_mask, hash_base, intlv_bit);
459 return inflated_addr | (hash_val << intlv_bit);
460 }
461
get_mchbar(struct pci_dev * pdev,u64 * mchbar)462 static int get_mchbar(struct pci_dev *pdev, u64 *mchbar)
463 {
464 union {
465 u64 v;
466 struct {
467 u32 v_lo;
468 u32 v_hi;
469 };
470 } u;
471
472 if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) {
473 igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n");
474 return -ENODEV;
475 }
476
477 if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) {
478 igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n");
479 return -ENODEV;
480 }
481
482 if (!(u.v & MCHBAR_EN)) {
483 igen6_printk(KERN_ERR, "MCHBAR is disabled\n");
484 return -ENODEV;
485 }
486
487 *mchbar = u.v & res_cfg->reg_mchbar_mask;
488 edac_dbg(2, "MCHBAR 0x%llx (reg 0x%llx)\n", *mchbar, u.v);
489
490 return 0;
491 }
492
493 /* Check whether the memory controller is absent. */
imc_absent(void __iomem * window)494 static bool imc_absent(void __iomem *window)
495 {
496 return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0;
497 }
498
499 /* Return MMIO base address of the memory controller if it's present, otherwise return NULL. */
map_imc_window(u64 mchbar,int pmc)500 static void __iomem *map_imc_window(u64 mchbar, int pmc)
501 {
502 void __iomem *window;
503
504 window = ioremap(mchbar + pmc * MCHBAR_SIZE, MCHBAR_SIZE);
505 if (!window)
506 return NULL;
507
508 if (imc_absent(window)) {
509 iounmap(window);
510 return NULL;
511 }
512
513 return window;
514 }
515
516 /* Return the number of present memory controllers. */
get_imc_num(u64 mchbar)517 static int get_imc_num(u64 mchbar)
518 {
519 void __iomem *window;
520 int lmc, pmc;
521
522 for (lmc = 0, pmc = 0; pmc < MAX_IMC_TO_PROBE; pmc++) {
523 window = map_imc_window(mchbar, pmc);
524 if (window) {
525 iounmap(window);
526 lmc++;
527 }
528 }
529
530 return lmc;
531 }
532
ehl_ibecc_available(struct pci_dev * pdev)533 static bool ehl_ibecc_available(struct pci_dev *pdev)
534 {
535 u32 v;
536
537 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
538 return false;
539
540 return !!(CAPID_C_IBECC & v);
541 }
542
ehl_err_addr_to_sys_addr(u64 eaddr,int mc)543 static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc)
544 {
545 return eaddr;
546 }
547
ehl_err_addr_to_imc_addr(u64 eaddr,int mc)548 static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc)
549 {
550 if (eaddr < igen6_tolud)
551 return eaddr;
552
553 if (igen6_tom <= _4GB)
554 return eaddr + igen6_tolud - _4GB;
555
556 if (eaddr >= igen6_tom)
557 return eaddr + igen6_tolud - igen6_tom;
558
559 return eaddr;
560 }
561
icl_ibecc_available(struct pci_dev * pdev)562 static bool icl_ibecc_available(struct pci_dev *pdev)
563 {
564 u32 v;
565
566 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
567 return false;
568
569 return !(CAPID_C_IBECC & v) &&
570 (boot_cpu_data.x86_stepping >= 1);
571 }
572
tgl_ibecc_available(struct pci_dev * pdev)573 static bool tgl_ibecc_available(struct pci_dev *pdev)
574 {
575 u32 v;
576
577 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
578 return false;
579
580 return !(CAPID_E_IBECC & v);
581 }
582
mtl_p_ibecc_available(struct pci_dev * pdev)583 static bool mtl_p_ibecc_available(struct pci_dev *pdev)
584 {
585 u32 v;
586
587 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
588 return false;
589
590 return !(CAPID_E_IBECC_BIT18 & v);
591 }
592
generic_ibecc_available(struct pci_dev * pdev)593 static bool generic_ibecc_available(struct pci_dev *pdev)
594 {
595 void __iomem *base = igen6_pvt->memss_pma_cr;
596 bool present;
597 u32 val;
598
599 if (res_cfg->reg_capabilities_misc_offset) {
600 val = readl(base + res_cfg->reg_capabilities_misc_offset);
601 present = !(val & res_cfg->reg_capabilities_misc_ibecc_dis);
602 edac_dbg(2, "capabilities misc reg 0x%x\n", val);
603 } else if (res_cfg->reg_mem_config_offset) {
604 val = readl(base + res_cfg->reg_mem_config_offset);
605 present = !!(val & res_cfg->reg_mem_config_ibecc_en_mask);
606 edac_dbg(2, "mem config reg 0x%x\n", val);
607 } else {
608 igen6_printk(KERN_ERR, "No register for detecting IBECC presence.\n");
609 present = false;
610 }
611
612 return present;
613 }
614
mem_addr_to_sys_addr(u64 maddr)615 static u64 mem_addr_to_sys_addr(u64 maddr)
616 {
617 if (maddr < igen6_tolud)
618 return maddr;
619
620 if (igen6_tom <= _4GB)
621 return maddr - igen6_tolud + _4GB;
622
623 if (maddr < _4GB)
624 return maddr - igen6_tolud + igen6_tom;
625
626 return maddr;
627 }
628
tgl_err_addr_to_mem_addr(u64 eaddr,int mc)629 static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc)
630 {
631 u64 mask, ms_s_size;
632 int intlv_bit;
633 u32 ms_hash;
634
635 ms_s_size = igen6_pvt->ms_s_size;
636 if (eaddr >= ms_s_size)
637 return eaddr + ms_s_size;
638
639 ms_hash = igen6_pvt->ms_hash;
640
641 mask = MEM_SLICE_HASH_MASK(ms_hash);
642 intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6;
643
644 return translate_to_upper_level(eaddr, mask, mc, intlv_bit, ms_s_size);
645 }
646
tgl_err_addr_to_sys_addr(u64 eaddr,int mc)647 static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc)
648 {
649 u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc);
650
651 return mem_addr_to_sys_addr(maddr);
652 }
653
tgl_err_addr_to_imc_addr(u64 eaddr,int mc)654 static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc)
655 {
656 return eaddr;
657 }
658
adl_err_addr_to_sys_addr(u64 eaddr,int mc)659 static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc)
660 {
661 return mem_addr_to_sys_addr(eaddr);
662 }
663
adl_err_addr_to_imc_addr(u64 eaddr,int mc)664 static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc)
665 {
666 u64 ms_s_size = igen6_pvt->ms_s_size;
667 struct igen6_imc *imc = &igen6_pvt->imc[mc];
668 struct slice slice;
669 int intlv_bit;
670 u32 mc_hash;
671
672 if (eaddr >= 2 * ms_s_size)
673 return eaddr - ms_s_size;
674
675 mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET);
676
677 intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6;
678
679 translate_to_lower_level(eaddr, 0, 0, intlv_bit, ms_s_size, 0, &slice);
680 return slice.addr;
681 }
682
ptl_h_get_mem_type(struct igen6_imc * imc)683 static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc)
684 {
685 u32 mtype, val;
686
687 val = readl(igen6_pvt->memss_pma_cr + res_cfg->reg_mem_config_offset);
688 mtype = field_get(res_cfg->reg_mem_config_ddr_type_mask, val);
689
690 edac_dbg(2, "mtype %u (reg 0x%x)\n", mtype, val);
691
692 switch (mtype) {
693 case 1:
694 return MEM_DDR5;
695 case 2:
696 return MEM_LPDDR5;
697 case 3:
698 return MEM_LPDDR4;
699 default:
700 return MEM_UNKNOWN;
701 }
702 }
703
ptl_h_get_dev_type(struct igen6_imc * imc,int chan,int dimm)704 static enum dev_type ptl_h_get_dev_type(struct igen6_imc *imc, int chan, int dimm)
705 {
706 u32 width, val;
707
708 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
709 width = field_get(res_cfg->reg_mad_intra_width_mask[dimm], val);
710
711 switch (width) {
712 case 1:
713 return DEV_X8;
714 default:
715 return DEV_X16;
716 }
717 }
718
ptl_h_get_chan_size(struct igen6_imc * imc,int chan)719 static u64 ptl_h_get_chan_size(struct igen6_imc *imc, int chan)
720 {
721 u32 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
722
723 return field_get(res_cfg->reg_mad_inter_size_mask[chan], val) *
724 res_cfg->reg_mad_inter_size_granularity;
725 }
726
ptl_h_get_dimm_size(struct igen6_imc * imc,int chan,int dimm)727 static u64 ptl_h_get_dimm_size(struct igen6_imc *imc, int chan, int dimm)
728 {
729 u32 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
730 u32 ranks = 1 << field_get(res_cfg->reg_mad_intra_rank_mask[dimm], val);
731 /* DRAM device density in Gb */
732 u64 density = field_get(res_cfg->reg_mad_intra_density_mask[dimm], val) * 4;
733
734 enum mem_type mtype = ptl_h_get_mem_type(imc);
735 enum dev_type dtype = ptl_h_get_dev_type(imc, chan, dimm);
736 u64 sub_ch_width, dev_num;
737
738 switch (mtype) {
739 case MEM_DDR5:
740 sub_ch_width = 32;
741 break;
742 case MEM_LPDDR5:
743 case MEM_LPDDR4:
744 sub_ch_width = 16;
745 break;
746 default:
747 sub_ch_width = 0;
748 }
749
750 switch (dtype) {
751 case DEV_X8:
752 dev_num = sub_ch_width / 8;
753 break;
754 case DEV_X16:
755 dev_num = sub_ch_width / 16;
756 break;
757 default:
758 dev_num = 0;
759 }
760
761 edac_dbg(2, "ranks %d, density %lluGb, sub_ch_width %llu, dev_num %llu (reg 0x%x)\n", ranks, density, sub_ch_width, dev_num, val);
762
763 return ((dev_num * density / 8) * ranks) << 30;
764 }
765
ptl_h_set_chan_params(struct igen6_imc * imc)766 static void ptl_h_set_chan_params(struct igen6_imc *imc)
767 {
768 u64 ch0_size = ptl_h_get_chan_size(imc, 0);
769 u64 ch1_size = ptl_h_get_chan_size(imc, 1);
770
771 if (ch0_size <= ch1_size) {
772 imc->ch_s_size = ch0_size;
773 imc->ch_l_map = 1;
774 } else {
775 imc->ch_s_size = ch1_size;
776 imc->ch_l_map = 0;
777 }
778 }
779
ptl_h_set_dimm_params(struct igen6_imc * imc,int chan)780 static void ptl_h_set_dimm_params(struct igen6_imc *imc, int chan)
781 {
782 u64 dimm0_size = ptl_h_get_dimm_size(imc, chan, 0);
783 u64 dimm1_size = ptl_h_get_dimm_size(imc, chan, 1);
784
785 if (dimm0_size <= dimm1_size) {
786 imc->dimm_s_size[chan] = dimm0_size;
787 imc->dimm_l_size[chan] = dimm1_size;
788 imc->dimm_l_map[chan] = 1;
789 } else {
790 imc->dimm_s_size[chan] = dimm1_size;
791 imc->dimm_l_size[chan] = dimm0_size;
792 imc->dimm_l_map[chan] = 0;
793 }
794 }
795
796 static struct res_config ehl_cfg = {
797 .num_imc = 1,
798 .reg_mchbar_mask = GENMASK_ULL(38, 16),
799 .reg_tom_mask = GENMASK_ULL(38, 20),
800 .reg_touud_mask = GENMASK_ULL(38, 20),
801 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
802 .imc_base = 0x5000,
803 .ibecc_base = 0xdc00,
804 .ibecc_available = ehl_ibecc_available,
805 .ibecc_error_log_offset = 0x170,
806 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr,
807 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr,
808 };
809
810 static struct res_config icl_cfg = {
811 .num_imc = 1,
812 .reg_mchbar_mask = GENMASK_ULL(38, 16),
813 .reg_tom_mask = GENMASK_ULL(38, 20),
814 .reg_touud_mask = GENMASK_ULL(38, 20),
815 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
816 .imc_base = 0x5000,
817 .ibecc_base = 0xd800,
818 .ibecc_error_log_offset = 0x170,
819 .ibecc_available = icl_ibecc_available,
820 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr,
821 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr,
822 };
823
824 static struct res_config tgl_cfg = {
825 .machine_check = true,
826 .num_imc = 2,
827 .reg_mchbar_mask = GENMASK_ULL(38, 17),
828 .reg_tom_mask = GENMASK_ULL(38, 20),
829 .reg_touud_mask = GENMASK_ULL(38, 20),
830 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
831 .imc_base = 0x5000,
832 .cmf_base = 0x11000,
833 .cmf_size = 0x800,
834 .ms_hash_offset = 0xac,
835 .ibecc_base = 0xd400,
836 .ibecc_error_log_offset = 0x170,
837 .ibecc_available = tgl_ibecc_available,
838 .err_addr_to_sys_addr = tgl_err_addr_to_sys_addr,
839 .err_addr_to_imc_addr = tgl_err_addr_to_imc_addr,
840 };
841
842 /* Shared by Alder Lake, Alder Lake-N, Arizona Beach, Amston Lake, and Raptor Lake-P */
843 static struct res_config adl_cfg = {
844 .machine_check = true,
845 .num_imc = 2,
846 .reg_mchbar_mask = GENMASK_ULL(41, 17),
847 .reg_tom_mask = GENMASK_ULL(41, 20),
848 .reg_touud_mask = GENMASK_ULL(41, 20),
849 .reg_eccerrlog_addr_mask = GENMASK_ULL(45, 5),
850 .imc_base = 0xd800,
851 .ibecc_base = 0xd400,
852 .ibecc_error_log_offset = 0x68,
853 .ibecc_available = tgl_ibecc_available,
854 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
855 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
856 };
857
858 static struct res_config mtl_ps_cfg = {
859 .machine_check = true,
860 .num_imc = 2,
861 .reg_mchbar_mask = GENMASK_ULL(41, 17),
862 .reg_tom_mask = GENMASK_ULL(41, 20),
863 .reg_touud_mask = GENMASK_ULL(41, 20),
864 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
865 .reg_capabilities_misc_offset = 0x13c00,
866 .reg_capabilities_misc_ibecc_dis = BIT(6),
867 .imc_base = 0xd800,
868 .ibecc_base = 0xd400,
869 .ibecc_error_log_offset = 0x170,
870 .ibecc_available = generic_ibecc_available,
871 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
872 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
873 };
874
875 /* Shared by Meteor Lake-P, Arrow Lake-UH, and Wildcat Lake */
876 static struct res_config mtl_p_cfg = {
877 .machine_check = true,
878 .num_imc = 2,
879 .reg_mchbar_mask = GENMASK_ULL(41, 17),
880 .reg_tom_mask = GENMASK_ULL(41, 20),
881 .reg_touud_mask = GENMASK_ULL(41, 20),
882 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
883 .imc_base = 0xd800,
884 .ibecc_base = 0xd400,
885 .ibecc_error_log_offset = 0x170,
886 .ibecc_available = mtl_p_ibecc_available,
887 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
888 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
889 };
890
891 /* Shared by Panther Lake-H and Starfire */
892 static struct res_config ptl_h_cfg = {
893 .machine_check = true,
894 .num_imc = 2,
895 .reg_mchbar_mask = GENMASK_ULL(41, 17),
896 .reg_tom_mask = GENMASK_ULL(41, 20),
897 .reg_touud_mask = GENMASK_ULL(41, 20),
898 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
899 .reg_mem_config_offset = 0x13d04,
900 .reg_mem_config_ddr_type_mask = GENMASK(8, 6),
901 .reg_mad_inter_size_mask[0] = GENMASK(15, 8),
902 .reg_mad_inter_size_mask[1] = GENMASK(23, 16),
903 .reg_mad_inter_size_granularity = BIT_ULL(29),
904 .reg_mad_intra_rank_mask[0] = BIT(7),
905 .reg_mad_intra_rank_mask[1] = BIT(15),
906 .reg_mad_intra_width_mask[0] = BIT(6),
907 .reg_mad_intra_width_mask[1] = BIT(14),
908 .reg_mad_intra_density_mask[0] = GENMASK(3, 0),
909 .reg_mad_intra_density_mask[1] = GENMASK(11, 8),
910 .imc_base = 0xd800,
911 .ibecc_base = 0xd400,
912 .ibecc_error_log_offset = 0x170,
913 .get_mem_type = ptl_h_get_mem_type,
914 .get_dev_type = ptl_h_get_dev_type,
915 .set_chan_params = ptl_h_set_chan_params,
916 .set_dimm_params = ptl_h_set_dimm_params,
917 .ibecc_available = mtl_p_ibecc_available,
918 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
919 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
920 };
921
922 static struct res_config nvl_h_cfg = {
923 .machine_check = true,
924 .num_imc = 2,
925 .reg_mchbar_mask = GENMASK_ULL(41, 17),
926 .reg_tom_mask = GENMASK_ULL(41, 20),
927 .reg_touud_mask = GENMASK_ULL(41, 20),
928 .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
929 .reg_mem_config_offset = 0x12904,
930 .reg_mem_config_ddr_type_mask = GENMASK(8, 6),
931 .reg_mem_config_ibecc_en_mask = GENMASK(3, 2),
932 .reg_mad_inter_size_mask[0] = GENMASK(15, 8),
933 .reg_mad_inter_size_mask[1] = GENMASK(23, 16),
934 .reg_mad_inter_size_granularity = BIT_ULL(29),
935 .reg_mad_intra_rank_mask[0] = BIT(7),
936 .reg_mad_intra_rank_mask[1] = BIT(15),
937 .reg_mad_intra_width_mask[0] = BIT(6),
938 .reg_mad_intra_width_mask[1] = BIT(14),
939 .reg_mad_intra_density_mask[0] = GENMASK(3, 0),
940 .reg_mad_intra_density_mask[1] = GENMASK(11, 8),
941 .imc_base = 0xd800,
942 .ibecc_base = 0xd400,
943 .ibecc_error_log_offset = 0x170,
944 .get_mem_type = ptl_h_get_mem_type,
945 .get_dev_type = ptl_h_get_dev_type,
946 .set_chan_params = ptl_h_set_chan_params,
947 .set_dimm_params = ptl_h_set_dimm_params,
948 .ibecc_available = generic_ibecc_available,
949 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
950 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
951 };
952
953 static struct pci_device_id igen6_pci_tbl[] = {
954 { PCI_VDEVICE(INTEL, DID_EHL_SKU5), .driver_data = (kernel_ulong_t)&ehl_cfg },
955 { PCI_VDEVICE(INTEL, DID_EHL_SKU6), .driver_data = (kernel_ulong_t)&ehl_cfg },
956 { PCI_VDEVICE(INTEL, DID_EHL_SKU7), .driver_data = (kernel_ulong_t)&ehl_cfg },
957 { PCI_VDEVICE(INTEL, DID_EHL_SKU8), .driver_data = (kernel_ulong_t)&ehl_cfg },
958 { PCI_VDEVICE(INTEL, DID_EHL_SKU9), .driver_data = (kernel_ulong_t)&ehl_cfg },
959 { PCI_VDEVICE(INTEL, DID_EHL_SKU10), .driver_data = (kernel_ulong_t)&ehl_cfg },
960 { PCI_VDEVICE(INTEL, DID_EHL_SKU11), .driver_data = (kernel_ulong_t)&ehl_cfg },
961 { PCI_VDEVICE(INTEL, DID_EHL_SKU12), .driver_data = (kernel_ulong_t)&ehl_cfg },
962 { PCI_VDEVICE(INTEL, DID_EHL_SKU13), .driver_data = (kernel_ulong_t)&ehl_cfg },
963 { PCI_VDEVICE(INTEL, DID_EHL_SKU14), .driver_data = (kernel_ulong_t)&ehl_cfg },
964 { PCI_VDEVICE(INTEL, DID_EHL_SKU15), .driver_data = (kernel_ulong_t)&ehl_cfg },
965 { PCI_VDEVICE(INTEL, DID_ICL_SKU8), .driver_data = (kernel_ulong_t)&icl_cfg },
966 { PCI_VDEVICE(INTEL, DID_ICL_SKU10), .driver_data = (kernel_ulong_t)&icl_cfg },
967 { PCI_VDEVICE(INTEL, DID_ICL_SKU11), .driver_data = (kernel_ulong_t)&icl_cfg },
968 { PCI_VDEVICE(INTEL, DID_ICL_SKU12), .driver_data = (kernel_ulong_t)&icl_cfg },
969 { PCI_VDEVICE(INTEL, DID_TGL_SKU), .driver_data = (kernel_ulong_t)&tgl_cfg },
970 { PCI_VDEVICE(INTEL, DID_ADL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
971 { PCI_VDEVICE(INTEL, DID_ADL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
972 { PCI_VDEVICE(INTEL, DID_ADL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
973 { PCI_VDEVICE(INTEL, DID_ADL_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg },
974 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
975 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
976 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
977 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg },
978 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), .driver_data = (kernel_ulong_t)&adl_cfg },
979 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), .driver_data = (kernel_ulong_t)&adl_cfg },
980 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), .driver_data = (kernel_ulong_t)&adl_cfg },
981 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), .driver_data = (kernel_ulong_t)&adl_cfg },
982 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), .driver_data = (kernel_ulong_t)&adl_cfg },
983 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), .driver_data = (kernel_ulong_t)&adl_cfg },
984 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), .driver_data = (kernel_ulong_t)&adl_cfg },
985 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), .driver_data = (kernel_ulong_t)&adl_cfg },
986 { PCI_VDEVICE(INTEL, DID_AZB_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
987 { PCI_VDEVICE(INTEL, DID_ASL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
988 { PCI_VDEVICE(INTEL, DID_ASL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
989 { PCI_VDEVICE(INTEL, DID_ASL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
990 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
991 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
992 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
993 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg },
994 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), .driver_data = (kernel_ulong_t)&adl_cfg },
995 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
996 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
997 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
998 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
999 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
1000 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
1001 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
1002 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
1003 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
1004 { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
1005 { PCI_VDEVICE(INTEL, DID_WCL_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
1006 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1007 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1008 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1009 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1010 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1011 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1012 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1013 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1014 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1015 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1016 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1017 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1018 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1019 { PCI_VDEVICE(INTEL, DID_PTL_H_SKU14), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1020 { PCI_VDEVICE(INTEL, DID_STF_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
1021 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU1), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
1022 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU2), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
1023 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU3), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
1024 { PCI_VDEVICE(INTEL, DID_NVL_H_SKU4), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
1025 { },
1026 };
1027 MODULE_DEVICE_TABLE(pci, igen6_pci_tbl);
1028
get_mem_type(struct igen6_imc * imc)1029 static enum mem_type get_mem_type(struct igen6_imc *imc)
1030 {
1031 u32 val;
1032
1033 if (res_cfg->get_mem_type)
1034 return res_cfg->get_mem_type(imc);
1035
1036 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
1037
1038 switch (MAD_INTER_CHANNEL_DDR_TYPE(val)) {
1039 case 0:
1040 return MEM_DDR4;
1041 case 1:
1042 return MEM_DDR3;
1043 case 2:
1044 return MEM_LPDDR3;
1045 case 3:
1046 return MEM_LPDDR4;
1047 case 4:
1048 return MEM_WIO2;
1049 default:
1050 return MEM_UNKNOWN;
1051 }
1052 }
1053
large_dimm(struct igen6_imc * imc,int chan,int dimm)1054 static bool large_dimm(struct igen6_imc *imc, int chan, int dimm)
1055 {
1056 return dimm == imc->dimm_l_map[chan];
1057 }
1058
get_dev_type(struct igen6_imc * imc,int chan,int dimm)1059 static enum dev_type get_dev_type(struct igen6_imc *imc, int chan, int dimm)
1060 {
1061 u32 width, val;
1062
1063 if (res_cfg->get_dev_type)
1064 return res_cfg->get_dev_type(imc, chan, dimm);
1065
1066 val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4);
1067 width = large_dimm(imc, chan, dimm) ? MAD_DIMM_CH_DLW(val) :
1068 MAD_DIMM_CH_DSW(val);
1069
1070 switch (width) {
1071 case 0:
1072 return DEV_X8;
1073 case 1:
1074 return DEV_X16;
1075 case 2:
1076 return DEV_X32;
1077 default:
1078 return DEV_UNKNOWN;
1079 }
1080 }
1081
get_dimm_size(struct igen6_imc * imc,int chan,int dimm)1082 static u64 get_dimm_size(struct igen6_imc *imc, int chan, int dimm)
1083 {
1084 if (large_dimm(imc, chan, dimm))
1085 return imc->dimm_l_size[chan];
1086
1087 return imc->dimm_s_size[chan];
1088 }
1089
set_chan_params(struct igen6_imc * imc)1090 static void set_chan_params(struct igen6_imc *imc)
1091 {
1092 u32 val;
1093
1094 if (res_cfg->set_chan_params) {
1095 res_cfg->set_chan_params(imc);
1096 return;
1097 }
1098
1099 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
1100 imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(val);
1101 imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(val);
1102 }
1103
set_dimm_params(struct igen6_imc * imc,int chan)1104 static void set_dimm_params(struct igen6_imc *imc, int chan)
1105 {
1106 u32 val;
1107
1108 if (res_cfg->set_dimm_params) {
1109 res_cfg->set_dimm_params(imc, chan);
1110 return;
1111 }
1112
1113 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
1114 imc->dimm_l_map[chan] = MAD_INTRA_CH_DIMM_L_MAP(val);
1115
1116 val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4);
1117 imc->dimm_l_size[chan] = MAD_DIMM_CH_DIMM_L_SIZE(val);
1118 imc->dimm_s_size[chan] = MAD_DIMM_CH_DIMM_S_SIZE(val);
1119 }
1120
igen6_decode(struct decoded_addr * res)1121 static int igen6_decode(struct decoded_addr *res)
1122 {
1123 struct igen6_imc *imc = &igen6_pvt->imc[res->mc];
1124 u64 addr = res->imc_addr, s_size;
1125 int intlv_bit, l_map;
1126 u32 hash, hash_mask;
1127 struct slice slice;
1128
1129 if (addr >= igen6_tom) {
1130 edac_dbg(0, "Address 0x%llx out of range\n", addr);
1131 return -EINVAL;
1132 }
1133
1134 /* Decode channel */
1135 hash = readl(imc->window + CHANNEL_HASH_OFFSET);
1136 s_size = imc->ch_s_size;
1137 l_map = imc->ch_l_map;
1138 hash_mask = CHANNEL_HASH_MODE(hash) ? CHANNEL_HASH_MASK(hash) : 0;
1139 intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6;
1140
1141 translate_to_lower_level(addr, hash_mask, 0, intlv_bit, s_size, l_map, &slice);
1142
1143 res->channel_idx = slice.id;
1144 res->channel_addr = slice.addr;
1145
1146 /* Decode sub-channel/DIMM */
1147 hash = readl(imc->window + CHANNEL_EHASH_OFFSET);
1148 s_size = imc->dimm_s_size[res->channel_idx];
1149 l_map = imc->dimm_l_map[res->channel_idx];
1150 hash_mask = CHANNEL_HASH_MODE(hash) ? CHANNEL_HASH_MASK(hash) : 0;
1151 intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6;
1152
1153 translate_to_lower_level(res->channel_addr, hash_mask, 0, intlv_bit, s_size, l_map, &slice);
1154
1155 res->sub_channel_idx = slice.id;
1156 res->sub_channel_addr = slice.addr;
1157
1158 return 0;
1159 }
1160
igen6_output_error(struct decoded_addr * res,struct mem_ctl_info * mci,u64 ecclog)1161 static void igen6_output_error(struct decoded_addr *res,
1162 struct mem_ctl_info *mci, u64 ecclog)
1163 {
1164 enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ?
1165 HW_EVENT_ERR_UNCORRECTED :
1166 HW_EVENT_ERR_CORRECTED;
1167
1168 edac_mc_handle_error(type, mci, 1,
1169 res->sys_addr >> PAGE_SHIFT,
1170 res->sys_addr & ~PAGE_MASK,
1171 ECC_ERROR_LOG_SYND(ecclog),
1172 res->channel_idx, res->sub_channel_idx,
1173 -1, "", "");
1174 }
1175
ecclog_gen_pool_create(void)1176 static struct gen_pool *ecclog_gen_pool_create(void)
1177 {
1178 struct gen_pool *pool;
1179
1180 pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1);
1181 if (!pool)
1182 return NULL;
1183
1184 if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) {
1185 gen_pool_destroy(pool);
1186 return NULL;
1187 }
1188
1189 return pool;
1190 }
1191
ecclog_gen_pool_add(int mc,u64 ecclog)1192 static int ecclog_gen_pool_add(int mc, u64 ecclog)
1193 {
1194 struct ecclog_node *node;
1195
1196 node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node));
1197 if (!node)
1198 return -ENOMEM;
1199
1200 node->mc = mc;
1201 node->ecclog = ecclog;
1202 llist_add(&node->llnode, &ecclog_llist);
1203
1204 return 0;
1205 }
1206
1207 /*
1208 * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI
1209 * configuration space status register ERRSTS can indicate whether a
1210 * correctable error or an uncorrectable error occurred. We only use the
1211 * ECC_ERROR_LOG register to check error type, but need to clear both
1212 * registers to enable future error events.
1213 */
ecclog_read_and_clear(struct igen6_imc * imc)1214 static u64 ecclog_read_and_clear(struct igen6_imc *imc)
1215 {
1216 u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET);
1217
1218 /*
1219 * Quirk: The ECC_ERROR_LOG register of certain SoCs may contain
1220 * the invalid value ~0. This will result in a flood of invalid
1221 * error reports in polling mode. Skip it.
1222 */
1223 if (ecclog == ~0)
1224 return 0;
1225
1226 /* Neither a CE nor a UE. Skip it.*/
1227 if (!(ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)))
1228 return 0;
1229
1230 /* Clear CE/UE bits by writing 1s */
1231 writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET);
1232
1233 return ecclog;
1234 }
1235
errsts_clear(struct igen6_imc * imc)1236 static void errsts_clear(struct igen6_imc *imc)
1237 {
1238 u16 errsts;
1239
1240 if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) {
1241 igen6_printk(KERN_ERR, "Failed to read ERRSTS\n");
1242 return;
1243 }
1244
1245 /* Clear CE/UE bits by writing 1s */
1246 if (errsts & (ERRSTS_CE | ERRSTS_UE))
1247 pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts);
1248 }
1249
errcmd_enable_error_reporting(bool enable)1250 static int errcmd_enable_error_reporting(bool enable)
1251 {
1252 struct igen6_imc *imc = &igen6_pvt->imc[0];
1253 u16 errcmd;
1254 int rc;
1255
1256 rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd);
1257 if (rc)
1258 return pcibios_err_to_errno(rc);
1259
1260 if (enable)
1261 errcmd |= ERRCMD_CE | ERRSTS_UE;
1262 else
1263 errcmd &= ~(ERRCMD_CE | ERRSTS_UE);
1264
1265 rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd);
1266 if (rc)
1267 return pcibios_err_to_errno(rc);
1268
1269 return 0;
1270 }
1271
ecclog_handler(void)1272 static int ecclog_handler(void)
1273 {
1274 struct igen6_imc *imc;
1275 int i, n = 0;
1276 u64 ecclog;
1277
1278 for (i = 0; i < res_cfg->num_imc; i++) {
1279 imc = &igen6_pvt->imc[i];
1280
1281 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
1282
1283 ecclog = ecclog_read_and_clear(imc);
1284 if (!ecclog)
1285 continue;
1286
1287 if (!ecclog_gen_pool_add(i, ecclog))
1288 irq_work_queue(&ecclog_irq_work);
1289
1290 n++;
1291 }
1292
1293 return n;
1294 }
1295
ecclog_work_cb(struct work_struct * work)1296 static void ecclog_work_cb(struct work_struct *work)
1297 {
1298 struct ecclog_node *node, *tmp;
1299 struct mem_ctl_info *mci;
1300 struct llist_node *head;
1301 struct decoded_addr res;
1302 u64 eaddr;
1303
1304 head = llist_del_all(&ecclog_llist);
1305 if (!head)
1306 return;
1307
1308 llist_for_each_entry_safe(node, tmp, head, llnode) {
1309 memset(&res, 0, sizeof(res));
1310 eaddr = node->ecclog & res_cfg->reg_eccerrlog_addr_mask;
1311 res.mc = node->mc;
1312 res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc);
1313 res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc);
1314
1315 mci = igen6_pvt->imc[res.mc].mci;
1316
1317 edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog);
1318 igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n");
1319 igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr);
1320
1321 if (!igen6_decode(&res))
1322 igen6_output_error(&res, mci, node->ecclog);
1323
1324 gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node));
1325 }
1326 }
1327
ecclog_irq_work_cb(struct irq_work * irq_work)1328 static void ecclog_irq_work_cb(struct irq_work *irq_work)
1329 {
1330 int i;
1331
1332 for (i = 0; i < res_cfg->num_imc; i++)
1333 errsts_clear(&igen6_pvt->imc[i]);
1334
1335 if (!llist_empty(&ecclog_llist))
1336 schedule_work(&ecclog_work);
1337 }
1338
ecclog_nmi_handler(unsigned int cmd,struct pt_regs * regs)1339 static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1340 {
1341 unsigned char reason;
1342
1343 if (!ecclog_handler())
1344 return NMI_DONE;
1345
1346 /*
1347 * Both In-Band ECC correctable error and uncorrectable error are
1348 * reported by SERR# NMI. The NMI generic code (see pci_serr_error())
1349 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to
1350 * re-enable the SERR# NMI after NMI handling. So clear this bit here
1351 * to re-enable SERR# NMI for receiving future In-Band ECC errors.
1352 */
1353 reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK;
1354 reason |= NMI_REASON_CLEAR_SERR;
1355 outb(reason, NMI_REASON_PORT);
1356 reason &= ~NMI_REASON_CLEAR_SERR;
1357 outb(reason, NMI_REASON_PORT);
1358
1359 return NMI_HANDLED;
1360 }
1361
ecclog_mce_handler(struct notifier_block * nb,unsigned long val,void * data)1362 static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val,
1363 void *data)
1364 {
1365 struct mce *mce = (struct mce *)data;
1366 char *type;
1367
1368 if (mce->kflags & MCE_HANDLED_CEC)
1369 return NOTIFY_DONE;
1370
1371 /*
1372 * Ignore unless this is a memory related error.
1373 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here,
1374 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3).
1375 */
1376 if ((mce->status & 0xefff) >> 7 != 1)
1377 return NOTIFY_DONE;
1378
1379 if (mce->mcgstatus & MCG_STATUS_MCIP)
1380 type = "Exception";
1381 else
1382 type = "Event";
1383
1384 edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n",
1385 mce->extcpu, type, mce->mcgstatus,
1386 mce->bank, mce->status);
1387 edac_dbg(0, "TSC 0x%llx\n", mce->tsc);
1388 edac_dbg(0, "ADDR 0x%llx\n", mce->addr);
1389 edac_dbg(0, "MISC 0x%llx\n", mce->misc);
1390 edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n",
1391 mce->cpuvendor, mce->cpuid, mce->time,
1392 mce->socketid, mce->apicid);
1393 /*
1394 * We just use the Machine Check for the memory error notification.
1395 * Each memory controller is associated with an IBECC instance.
1396 * Directly read and clear the error information(error address and
1397 * error type) on all the IBECC instances so that we know on which
1398 * memory controller the memory error(s) occurred.
1399 */
1400 if (!ecclog_handler())
1401 return NOTIFY_DONE;
1402
1403 mce->kflags |= MCE_HANDLED_EDAC;
1404
1405 return NOTIFY_DONE;
1406 }
1407
1408 static struct notifier_block ecclog_mce_dec = {
1409 .notifier_call = ecclog_mce_handler,
1410 .priority = MCE_PRIO_EDAC,
1411 };
1412
igen6_check_ecc(struct igen6_imc * imc)1413 static bool igen6_check_ecc(struct igen6_imc *imc)
1414 {
1415 u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET);
1416
1417 return !!(activate & IBECC_ACTIVATE_EN);
1418 }
1419
igen6_get_dimm_config(struct mem_ctl_info * mci)1420 static int igen6_get_dimm_config(struct mem_ctl_info *mci)
1421 {
1422 struct igen6_imc *imc = mci->pvt_info;
1423 int i, j, ndimms, mc = imc->mc;
1424 struct dimm_info *dimm;
1425 enum mem_type mtype;
1426 enum dev_type dtype;
1427 u64 dsize;
1428 bool ecc;
1429
1430 edac_dbg(2, "\n");
1431
1432 mtype = get_mem_type(imc);
1433 ecc = igen6_check_ecc(imc);
1434 set_chan_params(imc);
1435
1436 for (i = 0; i < NUM_CHANNELS; i++) {
1437 set_dimm_params(imc, i);
1438 imc->size += imc->dimm_s_size[i];
1439 imc->size += imc->dimm_l_size[i];
1440 ndimms = 0;
1441
1442 for (j = 0; j < NUM_DIMMS; j++) {
1443 dimm = edac_get_dimm(mci, i, j, 0);
1444 dtype = get_dev_type(imc, i, j);
1445 dsize = get_dimm_size(imc, i, j);
1446
1447 if (!dsize)
1448 continue;
1449
1450 dimm->grain = 64;
1451 dimm->mtype = mtype;
1452 dimm->dtype = dtype;
1453 dimm->nr_pages = MiB_TO_PAGES(dsize >> 20);
1454 dimm->edac_mode = EDAC_SECDED;
1455 snprintf(dimm->label, sizeof(dimm->label),
1456 "MC#%d_Chan#%d_DIMM#%d", mc, i, j);
1457 edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n",
1458 mc, i, j, dsize >> 20, dimm->nr_pages);
1459
1460 ndimms++;
1461 }
1462
1463 if (ndimms && !ecc) {
1464 igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc);
1465 return -ENODEV;
1466 }
1467 }
1468
1469 edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20);
1470
1471 return 0;
1472 }
1473
1474 #ifdef CONFIG_EDAC_DEBUG
1475 /* Top of upper usable DRAM */
1476 static u64 igen6_touud;
1477 #define TOUUD_OFFSET 0xa8
1478
igen6_reg_dump(struct igen6_imc * imc)1479 static void igen6_reg_dump(struct igen6_imc *imc)
1480 {
1481 int i;
1482
1483 edac_dbg(2, "CHANNEL_HASH : 0x%x\n",
1484 readl(imc->window + CHANNEL_HASH_OFFSET));
1485 edac_dbg(2, "CHANNEL_EHASH : 0x%x\n",
1486 readl(imc->window + CHANNEL_EHASH_OFFSET));
1487 edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n",
1488 readl(imc->window + MAD_INTER_CHANNEL_OFFSET));
1489 edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n",
1490 readq(imc->window + ECC_ERROR_LOG_OFFSET));
1491
1492 for (i = 0; i < NUM_CHANNELS; i++) {
1493 edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i,
1494 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4));
1495 edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i,
1496 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4));
1497 }
1498 edac_dbg(2, "TOLUD : 0x%x", igen6_tolud);
1499 edac_dbg(2, "TOUUD : 0x%llx", igen6_touud);
1500 edac_dbg(2, "TOM : 0x%llx", igen6_tom);
1501 }
1502
1503 static struct dentry *igen6_test;
1504
debugfs_u64_set(void * data,u64 val)1505 static int debugfs_u64_set(void *data, u64 val)
1506 {
1507 u64 ecclog;
1508
1509 if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) {
1510 edac_dbg(0, "Address 0x%llx out of range\n", val);
1511 return 0;
1512 }
1513
1514 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val);
1515
1516 ecclog = (val & res_cfg->reg_eccerrlog_addr_mask) | ECC_ERROR_LOG_CE;
1517
1518 if (!ecclog_gen_pool_add(0, ecclog))
1519 irq_work_queue(&ecclog_irq_work);
1520
1521 return 0;
1522 }
1523 DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
1524
igen6_debug_setup(void)1525 static void igen6_debug_setup(void)
1526 {
1527 igen6_test = edac_debugfs_create_dir("igen6_test");
1528 if (!igen6_test)
1529 return;
1530
1531 if (!edac_debugfs_create_file("addr", 0200, igen6_test,
1532 NULL, &fops_u64_wo)) {
1533 debugfs_remove(igen6_test);
1534 igen6_test = NULL;
1535 }
1536 }
1537
igen6_debug_teardown(void)1538 static void igen6_debug_teardown(void)
1539 {
1540 debugfs_remove_recursive(igen6_test);
1541 }
1542 #else
igen6_reg_dump(struct igen6_imc * imc)1543 static void igen6_reg_dump(struct igen6_imc *imc) {}
igen6_debug_setup(void)1544 static void igen6_debug_setup(void) {}
igen6_debug_teardown(void)1545 static void igen6_debug_teardown(void) {}
1546 #endif
1547
igen6_pvt_setup(struct pci_dev * pdev)1548 static struct igen6_pvt *igen6_pvt_setup(struct pci_dev *pdev)
1549 {
1550 void __iomem *memss_pma_cr;
1551 struct igen6_pvt *pvt;
1552 int imc_num, rc;
1553 u64 mchbar;
1554
1555 rc = get_mchbar(pdev, &mchbar);
1556 if (rc)
1557 return NULL;
1558
1559 imc_num = get_imc_num(mchbar);
1560 if (!imc_num) {
1561 igen6_printk(KERN_ERR, "No mc found.\n");
1562 return NULL;
1563 }
1564 edac_dbg(2, "%d mcs found.\n", imc_num);
1565
1566 /* Use the runtime detected IMC count. */
1567 if (res_cfg->num_imc != imc_num)
1568 res_cfg->num_imc = imc_num;
1569
1570 pvt = kzalloc_flex(*pvt, imc, imc_num);
1571 if (!pvt)
1572 return NULL;
1573
1574 memss_pma_cr = ioremap(mchbar, MCHBAR_SIZE * 2);
1575 if (!memss_pma_cr) {
1576 kfree(pvt);
1577 return NULL;
1578 }
1579 pvt->memss_pma_cr = memss_pma_cr;
1580
1581 return pvt;
1582 }
1583
igen6_pvt_release(struct igen6_pvt * pvt)1584 static void igen6_pvt_release(struct igen6_pvt *pvt)
1585 {
1586 iounmap(pvt->memss_pma_cr);
1587 kfree(pvt);
1588 }
1589
igen6_pci_setup(struct pci_dev * pdev,u64 * mchbar)1590 static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar)
1591 {
1592 union {
1593 u64 v;
1594 struct {
1595 u32 v_lo;
1596 u32 v_hi;
1597 };
1598 } u;
1599
1600 edac_dbg(2, "\n");
1601
1602 if (!res_cfg->ibecc_available(pdev)) {
1603 edac_dbg(2, "No In-Band ECC IP\n");
1604 goto fail;
1605 }
1606
1607 if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) {
1608 igen6_printk(KERN_ERR, "Failed to read TOLUD\n");
1609 goto fail;
1610 }
1611
1612 igen6_tolud &= GENMASK(31, 20);
1613
1614 if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) {
1615 igen6_printk(KERN_ERR, "Failed to read lower TOM\n");
1616 goto fail;
1617 }
1618
1619 if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) {
1620 igen6_printk(KERN_ERR, "Failed to read upper TOM\n");
1621 goto fail;
1622 }
1623
1624 igen6_tom = u.v & res_cfg->reg_tom_mask;
1625
1626 if (get_mchbar(pdev, mchbar))
1627 goto fail;
1628
1629 #ifdef CONFIG_EDAC_DEBUG
1630 if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo))
1631 edac_dbg(2, "Failed to read lower TOUUD\n");
1632 else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi))
1633 edac_dbg(2, "Failed to read upper TOUUD\n");
1634 else
1635 igen6_touud = u.v & res_cfg->reg_touud_mask;
1636 #endif
1637
1638 return 0;
1639 fail:
1640 return -ENODEV;
1641 }
1642
igen6_check(struct mem_ctl_info * mci)1643 static void igen6_check(struct mem_ctl_info *mci)
1644 {
1645 struct igen6_imc *imc = mci->pvt_info;
1646 u64 ecclog;
1647
1648 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
1649 ecclog = ecclog_read_and_clear(imc);
1650 if (!ecclog)
1651 return;
1652
1653 if (!ecclog_gen_pool_add(imc->mc, ecclog))
1654 irq_work_queue(&ecclog_irq_work);
1655 }
1656
imc_release(struct device * dev)1657 static void imc_release(struct device *dev)
1658 {
1659 /* Nothing to do, the 'imc' owns the 'dev' and will also release it. */
1660 }
1661
igen6_register_mci(int mc,void __iomem * window,struct pci_dev * pdev)1662 static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev)
1663 {
1664 struct edac_mc_layer layers[2];
1665 struct mem_ctl_info *mci;
1666 struct igen6_imc *imc;
1667 int rc;
1668
1669 edac_dbg(2, "\n");
1670
1671 layers[0].type = EDAC_MC_LAYER_CHANNEL;
1672 layers[0].size = NUM_CHANNELS;
1673 layers[0].is_virt_csrow = false;
1674 layers[1].type = EDAC_MC_LAYER_SLOT;
1675 layers[1].size = NUM_DIMMS;
1676 layers[1].is_virt_csrow = true;
1677
1678 mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0);
1679 if (!mci) {
1680 rc = -ENOMEM;
1681 goto fail;
1682 }
1683
1684 mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc);
1685 if (!mci->ctl_name) {
1686 rc = -ENOMEM;
1687 goto fail2;
1688 }
1689
1690 mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4;
1691 mci->edac_ctl_cap = EDAC_FLAG_SECDED;
1692 mci->edac_cap = EDAC_FLAG_SECDED;
1693 mci->mod_name = EDAC_MOD_STR;
1694 mci->dev_name = pci_name(pdev);
1695 if (edac_op_state == EDAC_OPSTATE_POLL)
1696 mci->edac_check = igen6_check;
1697 mci->pvt_info = &igen6_pvt->imc[mc];
1698
1699 imc = mci->pvt_info;
1700 imc->dev.release = imc_release;
1701 device_initialize(&imc->dev);
1702 /*
1703 * EDAC core uses mci->pdev(pointer of structure device) as
1704 * memory controller ID. The client SoCs attach one or more
1705 * memory controllers to single pci_dev (single pci_dev->dev
1706 * can be for multiple memory controllers).
1707 *
1708 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev
1709 * for the first memory controller and assign a unique imc->dev
1710 * to mci->pdev for each non-first memory controller.
1711 */
1712 mci->pdev = mc ? &imc->dev : &pdev->dev;
1713 imc->mc = mc;
1714 imc->pdev = pdev;
1715 imc->window = window;
1716
1717 igen6_reg_dump(imc);
1718
1719 rc = igen6_get_dimm_config(mci);
1720 if (rc)
1721 goto fail3;
1722
1723 rc = edac_mc_add_mc(mci);
1724 if (rc) {
1725 igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc);
1726 goto fail3;
1727 }
1728
1729 imc->mci = mci;
1730 return 0;
1731 fail3:
1732 put_device(&imc->dev);
1733 mci->pvt_info = NULL;
1734 kfree(mci->ctl_name);
1735 fail2:
1736 edac_mc_free(mci);
1737 fail:
1738 return rc;
1739 }
1740
igen6_unregister_mcis(void)1741 static void igen6_unregister_mcis(void)
1742 {
1743 struct mem_ctl_info *mci;
1744 struct igen6_imc *imc;
1745 int i;
1746
1747 edac_dbg(2, "\n");
1748
1749 for (i = 0; i < res_cfg->num_imc; i++) {
1750 imc = &igen6_pvt->imc[i];
1751 mci = imc->mci;
1752 if (!mci)
1753 continue;
1754
1755 edac_mc_del_mc(mci->pdev);
1756 kfree(mci->ctl_name);
1757 mci->pvt_info = NULL;
1758 edac_mc_free(mci);
1759 put_device(&imc->dev);
1760 iounmap(imc->window);
1761 }
1762 }
1763
igen6_register_mcis(struct pci_dev * pdev,u64 mchbar)1764 static int igen6_register_mcis(struct pci_dev *pdev, u64 mchbar)
1765 {
1766 void __iomem *window;
1767 int lmc, pmc, rc;
1768
1769 for (lmc = 0, pmc = 0; pmc < MAX_IMC_TO_PROBE; pmc++) {
1770 window = map_imc_window(mchbar, pmc);
1771 if (!window)
1772 continue;
1773
1774 rc = igen6_register_mci(lmc, window, pdev);
1775 if (rc)
1776 goto err_unregister;
1777
1778 /* Done, if all present MCs are detected and registered. */
1779 if (++lmc >= res_cfg->num_imc)
1780 break;
1781 }
1782
1783 if (!lmc) {
1784 igen6_printk(KERN_ERR, "No mc found.\n");
1785 return -ENODEV;
1786 }
1787
1788 if (lmc < res_cfg->num_imc) {
1789 igen6_printk(KERN_DEBUG, "Expected %d mcs, but only %d detected.",
1790 res_cfg->num_imc, lmc);
1791 res_cfg->num_imc = lmc;
1792 }
1793
1794 return 0;
1795
1796 err_unregister:
1797 iounmap(window);
1798 igen6_unregister_mcis();
1799
1800 return rc;
1801 }
1802
igen6_mem_slice_setup(u64 mchbar)1803 static int igen6_mem_slice_setup(u64 mchbar)
1804 {
1805 struct igen6_imc *imc = &igen6_pvt->imc[0];
1806 u64 base = mchbar + res_cfg->cmf_base;
1807 u32 offset = res_cfg->ms_hash_offset;
1808 u32 size = res_cfg->cmf_size;
1809 u64 ms_s_size, ms_hash;
1810 void __iomem *cmf;
1811 int ms_l_map;
1812
1813 edac_dbg(2, "\n");
1814
1815 if (imc[0].size < imc[1].size) {
1816 ms_s_size = imc[0].size;
1817 ms_l_map = 1;
1818 } else {
1819 ms_s_size = imc[1].size;
1820 ms_l_map = 0;
1821 }
1822
1823 igen6_pvt->ms_s_size = ms_s_size;
1824 igen6_pvt->ms_l_map = ms_l_map;
1825
1826 edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n",
1827 ms_s_size >> 20, ms_l_map);
1828
1829 if (!size)
1830 return 0;
1831
1832 cmf = ioremap(base, size);
1833 if (!cmf) {
1834 igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base);
1835 return -ENODEV;
1836 }
1837
1838 ms_hash = readq(cmf + offset);
1839 igen6_pvt->ms_hash = ms_hash;
1840
1841 edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash);
1842
1843 iounmap(cmf);
1844
1845 return 0;
1846 }
1847
register_err_handler(void)1848 static int register_err_handler(void)
1849 {
1850 int rc;
1851
1852 if (res_cfg->machine_check) {
1853 mce_register_decode_chain(&ecclog_mce_dec);
1854 return 0;
1855 }
1856
1857 rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler,
1858 0, IGEN6_NMI_NAME);
1859 if (rc) {
1860 igen6_printk(KERN_ERR, "Failed to register NMI handler\n");
1861 return rc;
1862 }
1863
1864 return 0;
1865 }
1866
unregister_err_handler(void)1867 static void unregister_err_handler(void)
1868 {
1869 if (res_cfg->machine_check) {
1870 mce_unregister_decode_chain(&ecclog_mce_dec);
1871 return;
1872 }
1873
1874 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1875 }
1876
opstate_set(const struct res_config * cfg,const struct pci_device_id * ent)1877 static void opstate_set(const struct res_config *cfg, const struct pci_device_id *ent)
1878 {
1879 /*
1880 * Quirk: Certain SoCs' error reporting interrupts don't work.
1881 * Force polling mode for them to ensure that memory error
1882 * events can be handled.
1883 */
1884 if (ent->device == DID_ADL_N_SKU4) {
1885 edac_op_state = EDAC_OPSTATE_POLL;
1886 return;
1887 }
1888
1889 /* Set the mode according to the configuration data. */
1890 if (cfg->machine_check)
1891 edac_op_state = EDAC_OPSTATE_INT;
1892 else
1893 edac_op_state = EDAC_OPSTATE_NMI;
1894 }
1895
igen6_probe(struct pci_dev * pdev,const struct pci_device_id * ent)1896 static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1897 {
1898 u64 mchbar;
1899 int rc;
1900
1901 edac_dbg(2, "\n");
1902
1903 res_cfg = (struct res_config *)ent->driver_data;
1904
1905 igen6_pvt = igen6_pvt_setup(pdev);
1906 if (!igen6_pvt)
1907 return -ENOMEM;
1908
1909 rc = igen6_pci_setup(pdev, &mchbar);
1910 if (rc)
1911 goto fail;
1912
1913 opstate_set(res_cfg, ent);
1914
1915 rc = igen6_register_mcis(pdev, mchbar);
1916 if (rc)
1917 goto fail;
1918
1919 if (res_cfg->num_imc > 1) {
1920 rc = igen6_mem_slice_setup(mchbar);
1921 if (rc)
1922 goto fail2;
1923 }
1924
1925 ecclog_pool = ecclog_gen_pool_create();
1926 if (!ecclog_pool) {
1927 rc = -ENOMEM;
1928 goto fail2;
1929 }
1930
1931 INIT_WORK(&ecclog_work, ecclog_work_cb);
1932 init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb);
1933
1934 rc = register_err_handler();
1935 if (rc)
1936 goto fail3;
1937
1938 /* Enable error reporting */
1939 rc = errcmd_enable_error_reporting(true);
1940 if (rc) {
1941 igen6_printk(KERN_ERR, "Failed to enable error reporting\n");
1942 goto fail4;
1943 }
1944
1945 /* Check if any pending errors before/during the registration of the error handler */
1946 ecclog_handler();
1947
1948 igen6_debug_setup();
1949 return 0;
1950 fail4:
1951 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1952 fail3:
1953 gen_pool_destroy(ecclog_pool);
1954 fail2:
1955 igen6_unregister_mcis();
1956 fail:
1957 igen6_pvt_release(igen6_pvt);
1958 return rc;
1959 }
1960
igen6_remove(struct pci_dev * pdev)1961 static void igen6_remove(struct pci_dev *pdev)
1962 {
1963 edac_dbg(2, "\n");
1964
1965 igen6_debug_teardown();
1966 errcmd_enable_error_reporting(false);
1967 unregister_err_handler();
1968 irq_work_sync(&ecclog_irq_work);
1969 flush_work(&ecclog_work);
1970 gen_pool_destroy(ecclog_pool);
1971 igen6_unregister_mcis();
1972 igen6_pvt_release(igen6_pvt);
1973 }
1974
1975 static struct pci_driver igen6_driver = {
1976 .name = EDAC_MOD_STR,
1977 .probe = igen6_probe,
1978 .remove = igen6_remove,
1979 .id_table = igen6_pci_tbl,
1980 };
1981
igen6_init(void)1982 static int __init igen6_init(void)
1983 {
1984 const char *owner;
1985 int rc;
1986
1987 edac_dbg(2, "\n");
1988
1989 if (ghes_get_devices())
1990 return -EBUSY;
1991
1992 owner = edac_get_owner();
1993 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
1994 return -EBUSY;
1995
1996 rc = pci_register_driver(&igen6_driver);
1997 if (rc)
1998 return rc;
1999
2000 igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION);
2001
2002 return 0;
2003 }
2004
igen6_exit(void)2005 static void __exit igen6_exit(void)
2006 {
2007 edac_dbg(2, "\n");
2008
2009 pci_unregister_driver(&igen6_driver);
2010 }
2011
2012 module_init(igen6_init);
2013 module_exit(igen6_exit);
2014
2015 MODULE_LICENSE("GPL v2");
2016 MODULE_AUTHOR("Qiuxu Zhuo");
2017 MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC");
2018