1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Driver for Intel(R) servers with Integrated Memory/IO Hub-based memory controller.
4 * Copyright (c) 2025, Intel Corporation.
5 */
6
7 #include <linux/kernel.h>
8 #include <linux/io.h>
9 #include <asm/cpu_device_id.h>
10 #include <asm/intel-family.h>
11 #include <asm/mce.h>
12 #include <asm/cpu.h>
13 #include "edac_module.h"
14 #include "skx_common.h"
15
16 #define IMH_REVISION "v0.0.1"
17 #define EDAC_MOD_STR "imh_edac"
18
19 /* Debug macros */
20 #define imh_printk(level, fmt, arg...) \
21 edac_printk(level, "imh", fmt, ##arg)
22
23 /* Configuration Agent(Ubox) */
24 #define MMIO_BASE_H(reg) (((u64)GET_BITFIELD(reg, 0, 29)) << 23)
25 #define SOCKET_ID(reg) GET_BITFIELD(reg, 0, 3)
26
27 /* PUNIT */
28 #define DDR_IMC_BITMAP(reg) GET_BITFIELD(reg, 23, 30)
29
30 /* Memory Controller */
31 #define ECC_ENABLED(reg) GET_BITFIELD(reg, 2, 2)
32 #define DIMM_POPULATED(reg) GET_BITFIELD(reg, 15, 15)
33
34 /* System Cache Agent(SCA) */
35 #define TOLM(reg) (((u64)GET_BITFIELD(reg, 16, 31)) << 16)
36 #define TOHM(reg) (((u64)GET_BITFIELD(reg, 16, 51)) << 16)
37
38 /* Home Agent (HA) */
39 #define NMCACHING(reg) GET_BITFIELD(reg, 8, 8)
40
41 /**
42 * struct local_reg - A register as described in the local package view.
43 *
44 * @pkg: (input) The package where the register is located.
45 * @pbase: (input) The IP MMIO base physical address in the local package view.
46 * @size: (input) The IP MMIO size.
47 * @offset: (input) The register offset from the IP MMIO base @pbase.
48 * @width: (input) The register width in byte.
49 * @vbase: (internal) The IP MMIO base virtual address.
50 * @val: (output) The register value.
51 */
52 struct local_reg {
53 int pkg;
54 u64 pbase;
55 u32 size;
56 u32 offset;
57 u8 width;
58 void __iomem *vbase;
59 u64 val;
60 };
61
62 #define DEFINE_LOCAL_REG(name, cfg, package, north, ip_name, ip_idx, reg_name) \
63 struct local_reg name = { \
64 .pkg = package, \
65 .pbase = (north ? (cfg)->mmio_base_l_north : \
66 (cfg)->mmio_base_l_south) + \
67 (cfg)->ip_name##_base + \
68 (cfg)->ip_name##_size * (ip_idx), \
69 .size = (cfg)->ip_name##_size, \
70 .offset = (cfg)->ip_name##_reg_##reg_name##_offset, \
71 .width = (cfg)->ip_name##_reg_##reg_name##_width, \
72 }
73
readx(void __iomem * addr,u8 width)74 static u64 readx(void __iomem *addr, u8 width)
75 {
76 switch (width) {
77 case 1:
78 return readb(addr);
79 case 2:
80 return readw(addr);
81 case 4:
82 return readl(addr);
83 case 8:
84 return readq(addr);
85 default:
86 imh_printk(KERN_ERR, "Invalid reg 0x%p width %d\n", addr, width);
87 return 0;
88 }
89 }
90
__read_local_reg(void * reg)91 static void __read_local_reg(void *reg)
92 {
93 struct local_reg *r = (struct local_reg *)reg;
94
95 r->val = readx(r->vbase + r->offset, r->width);
96 }
97
98 /* Read a local-view register. */
read_local_reg(struct local_reg * reg)99 static bool read_local_reg(struct local_reg *reg)
100 {
101 int cpu;
102
103 /* Get the target CPU in the package @reg->pkg. */
104 for_each_online_cpu(cpu) {
105 if (reg->pkg == topology_physical_package_id(cpu))
106 break;
107 }
108
109 if (cpu >= nr_cpu_ids)
110 return false;
111
112 reg->vbase = ioremap(reg->pbase, reg->size);
113 if (!reg->vbase) {
114 imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", reg->pbase);
115 return false;
116 }
117
118 /* Get the target CPU to read the register. */
119 smp_call_function_single(cpu, __read_local_reg, reg, 1);
120 iounmap(reg->vbase);
121
122 return true;
123 }
124
125 /* Get the bitmap of memory controller instances in package @pkg. */
get_imc_bitmap(struct res_config * cfg,int pkg,bool north)126 static u32 get_imc_bitmap(struct res_config *cfg, int pkg, bool north)
127 {
128 DEFINE_LOCAL_REG(reg, cfg, pkg, north, pcu, 0, capid3);
129
130 if (!read_local_reg(®))
131 return 0;
132
133 edac_dbg(2, "Pkg%d %s mc instances bitmap 0x%llx (reg 0x%llx)\n",
134 pkg, north ? "north" : "south",
135 DDR_IMC_BITMAP(reg.val), reg.val);
136
137 return DDR_IMC_BITMAP(reg.val);
138 }
139
imc_release(struct device * dev)140 static void imc_release(struct device *dev)
141 {
142 edac_dbg(2, "imc device %s released\n", dev_name(dev));
143 kfree(dev);
144 }
145
__get_ddr_munits(struct res_config * cfg,struct skx_dev * d,bool north,int lmc)146 static int __get_ddr_munits(struct res_config *cfg, struct skx_dev *d,
147 bool north, int lmc)
148 {
149 unsigned long size = cfg->ddr_chan_mmio_sz * cfg->ddr_chan_num;
150 unsigned long bitmap = get_imc_bitmap(cfg, d->pkg, north);
151 void __iomem *mbase;
152 struct device *dev;
153 int i, rc, pmc;
154 u64 base;
155
156 for_each_set_bit(i, &bitmap, sizeof(bitmap) * 8) {
157 base = north ? d->mmio_base_h_north : d->mmio_base_h_south;
158 base += cfg->ddr_imc_base + size * i;
159
160 edac_dbg(2, "Pkg%d mc%d mmio base 0x%llx size 0x%lx\n",
161 d->pkg, lmc, base, size);
162
163 /* Set up the imc MMIO. */
164 mbase = ioremap(base, size);
165 if (!mbase) {
166 imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", base);
167 return -ENOMEM;
168 }
169
170 d->imc[lmc].mbase = mbase;
171 d->imc[lmc].lmc = lmc;
172
173 /* Create the imc device instance. */
174 dev = kzalloc_obj(*dev);
175 if (!dev)
176 return -ENOMEM;
177
178 dev->release = imc_release;
179 device_initialize(dev);
180 rc = dev_set_name(dev, "0x%llx", base);
181 if (rc) {
182 imh_printk(KERN_ERR, "Failed to set dev name\n");
183 put_device(dev);
184 return rc;
185 }
186
187 d->imc[lmc].dev = dev;
188
189 /* Set up the imc index mapping. */
190 pmc = north ? i : 8 + i;
191 skx_set_mc_mapping(d, pmc, lmc);
192
193 lmc++;
194 }
195
196 return lmc;
197 }
198
get_ddr_munits(struct res_config * cfg,struct skx_dev * d)199 static bool get_ddr_munits(struct res_config *cfg, struct skx_dev *d)
200 {
201 int lmc = __get_ddr_munits(cfg, d, true, 0);
202
203 if (lmc < 0)
204 return false;
205
206 lmc = __get_ddr_munits(cfg, d, false, lmc);
207 if (lmc <= 0)
208 return false;
209
210 return true;
211 }
212
get_socket_id(struct res_config * cfg,struct skx_dev * d)213 static bool get_socket_id(struct res_config *cfg, struct skx_dev *d)
214 {
215 DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ubox, 0, socket_id);
216 u8 src_id;
217 int i;
218
219 if (!read_local_reg(®))
220 return false;
221
222 src_id = SOCKET_ID(reg.val);
223 edac_dbg(2, "socket id 0x%x (reg 0x%llx)\n", src_id, reg.val);
224
225 for (i = 0; i < cfg->ddr_imc_num; i++)
226 d->imc[i].src_id = src_id;
227
228 return true;
229 }
230
231 /* Get TOLM (Top Of Low Memory) and TOHM (Top Of High Memory) parameters. */
imh_get_tolm_tohm(struct res_config * cfg,u64 * tolm,u64 * tohm)232 static bool imh_get_tolm_tohm(struct res_config *cfg, u64 *tolm, u64 *tohm)
233 {
234 DEFINE_LOCAL_REG(reg, cfg, 0, true, sca, 0, tolm);
235
236 if (!read_local_reg(®))
237 return false;
238
239 *tolm = TOLM(reg.val);
240 edac_dbg(2, "tolm 0x%llx (reg 0x%llx)\n", *tolm, reg.val);
241
242 DEFINE_LOCAL_REG(reg2, cfg, 0, true, sca, 0, tohm);
243
244 if (!read_local_reg(®2))
245 return false;
246
247 *tohm = TOHM(reg2.val);
248 edac_dbg(2, "tohm 0x%llx (reg 0x%llx)\n", *tohm, reg2.val);
249
250 return true;
251 }
252
253 /* Get the system-view MMIO_BASE_H for {north,south}-IMH. */
imh_get_all_mmio_base_h(struct res_config * cfg,struct list_head * edac_list)254 static int imh_get_all_mmio_base_h(struct res_config *cfg, struct list_head *edac_list)
255 {
256 int i, n = topology_max_packages(), imc_num = cfg->ddr_imc_num + cfg->hbm_imc_num;
257 struct skx_dev *d;
258
259 for (i = 0; i < n; i++) {
260 d = kzalloc_flex(*d, imc, imc_num);
261 if (!d)
262 return -ENOMEM;
263
264 DEFINE_LOCAL_REG(reg, cfg, i, true, ubox, 0, mmio_base);
265
266 /* Get MMIO_BASE_H for the north-IMH. */
267 if (!read_local_reg(®) || !reg.val) {
268 kfree(d);
269 imh_printk(KERN_ERR, "Pkg%d has no north mmio_base_h\n", i);
270 return -ENODEV;
271 }
272
273 d->mmio_base_h_north = MMIO_BASE_H(reg.val);
274 edac_dbg(2, "Pkg%d north mmio_base_h 0x%llx (reg 0x%llx)\n",
275 i, d->mmio_base_h_north, reg.val);
276
277 /* Get MMIO_BASE_H for the south-IMH (optional). */
278 DEFINE_LOCAL_REG(reg2, cfg, i, false, ubox, 0, mmio_base);
279
280 if (read_local_reg(®2)) {
281 d->mmio_base_h_south = MMIO_BASE_H(reg2.val);
282 edac_dbg(2, "Pkg%d south mmio_base_h 0x%llx (reg 0x%llx)\n",
283 i, d->mmio_base_h_south, reg2.val);
284 }
285
286 d->pkg = i;
287 d->num_imc = imc_num;
288 skx_init_mc_mapping(d);
289 list_add_tail(&d->list, edac_list);
290 }
291
292 return 0;
293 }
294
295 /* Get the number of per-package memory controllers. */
imh_get_imc_num(struct res_config * cfg)296 static int imh_get_imc_num(struct res_config *cfg)
297 {
298 int imc_num = hweight32(get_imc_bitmap(cfg, 0, true)) +
299 hweight32(get_imc_bitmap(cfg, 0, false));
300
301 if (!imc_num) {
302 imh_printk(KERN_ERR, "Invalid mc number\n");
303 return -ENODEV;
304 }
305
306 if (cfg->ddr_imc_num != imc_num) {
307 /*
308 * Update the configuration data to reflect the number of
309 * present DDR memory controllers.
310 */
311 cfg->ddr_imc_num = imc_num;
312 edac_dbg(2, "Set ddr mc number %d\n", imc_num);
313 }
314
315 return 0;
316 }
317
318 /* Get all memory controllers' parameters. */
imh_get_munits(struct res_config * cfg,struct list_head * edac_list)319 static int imh_get_munits(struct res_config *cfg, struct list_head *edac_list)
320 {
321 struct skx_imc *imc;
322 struct skx_dev *d;
323 u8 mc = 0;
324 int i;
325
326 list_for_each_entry(d, edac_list, list) {
327 if (!get_ddr_munits(cfg, d)) {
328 imh_printk(KERN_ERR, "No mc found\n");
329 return -ENODEV;
330 }
331
332 if (!get_socket_id(cfg, d)) {
333 imh_printk(KERN_ERR, "Failed to get socket id\n");
334 return -ENODEV;
335 }
336
337 for (i = 0; i < cfg->ddr_imc_num; i++) {
338 imc = &d->imc[i];
339 if (!imc->mbase)
340 continue;
341
342 imc->chan_mmio_sz = cfg->ddr_chan_mmio_sz;
343 imc->num_channels = cfg->ddr_chan_num;
344 imc->num_dimms = cfg->ddr_dimm_num;
345 imc->mc = mc++;
346 }
347 }
348
349 return 0;
350 }
351
check_2lm_enabled(struct res_config * cfg,struct skx_dev * d,int ha_idx)352 static bool check_2lm_enabled(struct res_config *cfg, struct skx_dev *d, int ha_idx)
353 {
354 DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ha, ha_idx, mode);
355
356 if (!read_local_reg(®))
357 return false;
358
359 if (!NMCACHING(reg.val))
360 return false;
361
362 edac_dbg(2, "2-level memory configuration (reg 0x%llx, ha idx %d)\n", reg.val, ha_idx);
363 return true;
364 }
365
366 /* Check whether the system has a 2-level memory configuration. */
imh_2lm_enabled(struct res_config * cfg,struct list_head * head)367 static bool imh_2lm_enabled(struct res_config *cfg, struct list_head *head)
368 {
369 struct skx_dev *d;
370 int i;
371
372 list_for_each_entry(d, head, list) {
373 for (i = 0; i < cfg->ddr_imc_num; i++)
374 if (check_2lm_enabled(cfg, d, i))
375 return true;
376 }
377
378 return false;
379 }
380
381 /* Helpers to read memory controller registers */
read_imc_reg(struct skx_imc * imc,int chan,u32 offset,u8 width)382 static u64 read_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width)
383 {
384 return readx(imc->mbase + imc->chan_mmio_sz * chan + offset, width);
385 }
386
read_imc_mcmtr(struct res_config * cfg,struct skx_imc * imc,int chan)387 static u32 read_imc_mcmtr(struct res_config *cfg, struct skx_imc *imc, int chan)
388 {
389 return (u32)read_imc_reg(imc, chan, cfg->ddr_reg_mcmtr_offset, cfg->ddr_reg_mcmtr_width);
390 }
391
read_imc_dimmmtr(struct res_config * cfg,struct skx_imc * imc,int chan,int dimm)392 static u32 read_imc_dimmmtr(struct res_config *cfg, struct skx_imc *imc, int chan, int dimm)
393 {
394 return (u32)read_imc_reg(imc, chan, cfg->ddr_reg_dimmmtr_offset +
395 cfg->ddr_reg_dimmmtr_width * dimm,
396 cfg->ddr_reg_dimmmtr_width);
397 }
398
ecc_enabled(u32 mcmtr)399 static bool ecc_enabled(u32 mcmtr)
400 {
401 return (bool)ECC_ENABLED(mcmtr);
402 }
403
dimm_populated(u32 dimmmtr)404 static bool dimm_populated(u32 dimmmtr)
405 {
406 return (bool)DIMM_POPULATED(dimmmtr);
407 }
408
409 /* Get each DIMM's configurations of the memory controller @mci. */
imh_get_dimm_config(struct mem_ctl_info * mci,struct res_config * cfg)410 static int imh_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg)
411 {
412 struct skx_pvt *pvt = mci->pvt_info;
413 struct skx_imc *imc = pvt->imc;
414 struct dimm_info *dimm;
415 u32 mcmtr, dimmmtr;
416 int i, j, ndimms;
417
418 for (i = 0; i < imc->num_channels; i++) {
419 if (!imc->mbase)
420 continue;
421
422 mcmtr = read_imc_mcmtr(cfg, imc, i);
423
424 for (ndimms = 0, j = 0; j < imc->num_dimms; j++) {
425 dimmmtr = read_imc_dimmmtr(cfg, imc, i, j);
426 edac_dbg(1, "mcmtr 0x%x dimmmtr 0x%x (mc%d ch%d dimm%d)\n",
427 mcmtr, dimmmtr, imc->mc, i, j);
428
429 if (!dimm_populated(dimmmtr))
430 continue;
431
432 dimm = edac_get_dimm(mci, i, j, 0);
433 ndimms += skx_get_dimm_info(dimmmtr, 0, 0, dimm,
434 imc, i, j, cfg);
435 }
436
437 if (ndimms && !ecc_enabled(mcmtr)) {
438 imh_printk(KERN_ERR, "ECC is disabled on mc%d ch%d\n",
439 imc->mc, i);
440 return -ENODEV;
441 }
442 }
443
444 return 0;
445 }
446
447 /* Register all memory controllers to the EDAC core. */
imh_register_mci(struct res_config * cfg,struct list_head * edac_list)448 static int imh_register_mci(struct res_config *cfg, struct list_head *edac_list)
449 {
450 struct skx_imc *imc;
451 struct skx_dev *d;
452 int i, rc;
453
454 list_for_each_entry(d, edac_list, list) {
455 for (i = 0; i < cfg->ddr_imc_num; i++) {
456 imc = &d->imc[i];
457 if (!imc->mbase)
458 continue;
459
460 rc = skx_register_mci(imc, imc->dev,
461 dev_name(imc->dev),
462 "Intel IMH-based Socket",
463 EDAC_MOD_STR,
464 imh_get_dimm_config, cfg);
465 if (rc)
466 return rc;
467 }
468 }
469
470 return 0;
471 }
472
473 static struct res_config dmr_cfg = {
474 .type = DMR,
475 .support_ddr5 = true,
476 .mmio_base_l_north = 0xf6800000,
477 .mmio_base_l_south = 0xf6000000,
478 .ddr_chan_num = 1,
479 .ddr_dimm_num = 2,
480 .ddr_imc_base = 0x39b000,
481 .ddr_chan_mmio_sz = 0x8000,
482 .ddr_reg_mcmtr_offset = 0x360,
483 .ddr_reg_mcmtr_width = 4,
484 .ddr_reg_dimmmtr_offset = 0x370,
485 .ddr_reg_dimmmtr_width = 4,
486 .ubox_base = 0x0,
487 .ubox_size = 0x2000,
488 .ubox_reg_mmio_base_offset = 0x580,
489 .ubox_reg_mmio_base_width = 4,
490 .ubox_reg_socket_id_offset = 0x1080,
491 .ubox_reg_socket_id_width = 4,
492 .pcu_base = 0x3000,
493 .pcu_size = 0x10000,
494 .pcu_reg_capid3_offset = 0x290,
495 .pcu_reg_capid3_width = 4,
496 .sca_base = 0x24c000,
497 .sca_size = 0x2500,
498 .sca_reg_tolm_offset = 0x2100,
499 .sca_reg_tolm_width = 8,
500 .sca_reg_tohm_offset = 0x2108,
501 .sca_reg_tohm_width = 8,
502 .ha_base = 0x3eb000,
503 .ha_size = 0x1000,
504 .ha_reg_mode_offset = 0x4a0,
505 .ha_reg_mode_width = 4,
506 };
507
508 static const struct x86_cpu_id imh_cpuids[] = {
509 X86_MATCH_VFM(INTEL_DIAMONDRAPIDS_X, &dmr_cfg),
510 {}
511 };
512 MODULE_DEVICE_TABLE(x86cpu, imh_cpuids);
513
514 static struct notifier_block imh_mce_dec = {
515 .notifier_call = skx_mce_check_error,
516 .priority = MCE_PRIO_EDAC,
517 };
518
imh_init(void)519 static int __init imh_init(void)
520 {
521 const struct x86_cpu_id *id;
522 struct list_head *edac_list;
523 struct res_config *cfg;
524 const char *owner;
525 u64 tolm, tohm;
526 int rc;
527
528 edac_dbg(2, "\n");
529
530 if (ghes_get_devices())
531 return -EBUSY;
532
533 owner = edac_get_owner();
534 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
535 return -EBUSY;
536
537 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
538 return -ENODEV;
539
540 id = x86_match_cpu(imh_cpuids);
541 if (!id)
542 return -ENODEV;
543 cfg = (struct res_config *)id->driver_data;
544 skx_set_res_cfg(cfg);
545
546 if (!imh_get_tolm_tohm(cfg, &tolm, &tohm))
547 return -ENODEV;
548
549 skx_set_hi_lo(tolm, tohm);
550
551 rc = imh_get_imc_num(cfg);
552 if (rc < 0)
553 goto fail;
554
555 edac_list = skx_get_edac_list();
556
557 rc = imh_get_all_mmio_base_h(cfg, edac_list);
558 if (rc)
559 goto fail;
560
561 rc = imh_get_munits(cfg, edac_list);
562 if (rc)
563 goto fail;
564
565 skx_set_mem_cfg(imh_2lm_enabled(cfg, edac_list));
566
567 rc = imh_register_mci(cfg, edac_list);
568 if (rc)
569 goto fail;
570
571 rc = skx_adxl_get();
572 if (rc)
573 goto fail;
574
575 opstate_init();
576 mce_register_decode_chain(&imh_mce_dec);
577 skx_setup_debug("imh_test");
578
579 imh_printk(KERN_INFO, "%s\n", IMH_REVISION);
580
581 return 0;
582 fail:
583 skx_remove();
584 return rc;
585 }
586
imh_exit(void)587 static void __exit imh_exit(void)
588 {
589 edac_dbg(2, "\n");
590
591 skx_teardown_debug();
592 mce_unregister_decode_chain(&imh_mce_dec);
593 skx_adxl_put();
594 skx_remove();
595 }
596
597 module_init(imh_init);
598 module_exit(imh_exit);
599
600 MODULE_LICENSE("GPL");
601 MODULE_AUTHOR("Qiuxu Zhuo");
602 MODULE_DESCRIPTION("MC Driver for Intel servers using IMH-based memory controller");
603