1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Driver for Intel(R) servers with Integrated Memory/IO Hub-based memory controller. 4 * Copyright (c) 2025, Intel Corporation. 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/io.h> 9 #include <asm/cpu_device_id.h> 10 #include <asm/intel-family.h> 11 #include <asm/mce.h> 12 #include <asm/cpu.h> 13 #include "edac_module.h" 14 #include "skx_common.h" 15 16 #define IMH_REVISION "v0.0.1" 17 #define EDAC_MOD_STR "imh_edac" 18 19 /* Debug macros */ 20 #define imh_printk(level, fmt, arg...) \ 21 edac_printk(level, "imh", fmt, ##arg) 22 23 /* Configuration Agent(Ubox) */ 24 #define MMIO_BASE_H(reg) (((u64)GET_BITFIELD(reg, 0, 29)) << 23) 25 #define SOCKET_ID(reg) GET_BITFIELD(reg, 0, 3) 26 27 /* PUNIT */ 28 #define DDR_IMC_BITMAP(reg) GET_BITFIELD(reg, 23, 30) 29 30 /* Memory Controller */ 31 #define ECC_ENABLED(reg) GET_BITFIELD(reg, 2, 2) 32 #define DIMM_POPULATED(reg) GET_BITFIELD(reg, 15, 15) 33 34 /* System Cache Agent(SCA) */ 35 #define TOLM(reg) (((u64)GET_BITFIELD(reg, 16, 31)) << 16) 36 #define TOHM(reg) (((u64)GET_BITFIELD(reg, 16, 51)) << 16) 37 38 /* Home Agent (HA) */ 39 #define NMCACHING(reg) GET_BITFIELD(reg, 8, 8) 40 41 /** 42 * struct local_reg - A register as described in the local package view. 43 * 44 * @pkg: (input) The package where the register is located. 45 * @pbase: (input) The IP MMIO base physical address in the local package view. 46 * @size: (input) The IP MMIO size. 47 * @offset: (input) The register offset from the IP MMIO base @pbase. 48 * @width: (input) The register width in byte. 49 * @vbase: (internal) The IP MMIO base virtual address. 50 * @val: (output) The register value. 51 */ 52 struct local_reg { 53 int pkg; 54 u64 pbase; 55 u32 size; 56 u32 offset; 57 u8 width; 58 void __iomem *vbase; 59 u64 val; 60 }; 61 62 #define DEFINE_LOCAL_REG(name, cfg, package, north, ip_name, ip_idx, reg_name) \ 63 struct local_reg name = { \ 64 .pkg = package, \ 65 .pbase = (north ? (cfg)->mmio_base_l_north : \ 66 (cfg)->mmio_base_l_south) + \ 67 (cfg)->ip_name##_base + \ 68 (cfg)->ip_name##_size * (ip_idx), \ 69 .size = (cfg)->ip_name##_size, \ 70 .offset = (cfg)->ip_name##_reg_##reg_name##_offset, \ 71 .width = (cfg)->ip_name##_reg_##reg_name##_width, \ 72 } 73 74 static struct res_config *res_cfg; 75 static int retry_rd_err_log; 76 77 #define REG_RRL_DEFINE(a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3) \ 78 { \ 79 .set_num = 4, \ 80 .reg_num = 7, \ 81 .sources = {RRL_SRC_FRE_SCRUB, RRL_SRC_FRE_DEMAND, RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, \ 82 .offsets = { \ 83 {a0, a1, a2, a3, a4, a5, a6}, \ 84 {a0 + 4, a1 + 4, a2 + 8, a3 + 4, a4 + 4, a5 + 8, a6 + 8}, \ 85 {a0 + 8, a1 + 8, a2 + 16, a3 + 8, a4 + 8, a5 + 16, a6 + 16}, \ 86 {a0 + 12, a1 + 12, a2 + 24, a3 + 12, a4 + 12, a5 + 24, a6 + 24}, \ 87 }, \ 88 .widths = {4, 4, 8, 4, 4, 8, 8}, \ 89 .v_mask = BIT(0), \ 90 .uc_mask = BIT(1), \ 91 .over_mask = BIT(2), \ 92 .en_mask = BIT(12), \ 93 .en_patspr_mask = BIT(14), \ 94 .noover_mask = BIT(15), \ 95 .cecnt_num = 4, \ 96 .cecnt_offsets = {b0, b1, b2, b3}, \ 97 .cecnt_widths = {8, 8, 8, 8}, \ 98 } 99 100 static struct reg_rrl dmr_reg_rrl_ddr_subch0 = REG_RRL_DEFINE( 101 0x2dc0, 0x2dd0, 0x2de0, 0x2e00, 0x2e10, 0x2f70, 0x0200, 102 0x2c10, 0x2c18, 0x2c20, 0x2c28); 103 static struct reg_rrl dmr_reg_rrl_ddr_subch1 = REG_RRL_DEFINE( 104 0x6dc0, 0x6dd0, 0x6de0, 0x6e00, 0x6e10, 0x6f70, 0x4200, 105 0x6c10, 0x6c18, 0x6c20, 0x6c28); 106 107 static void __read_local_reg(void *reg) 108 { 109 struct local_reg *r = (struct local_reg *)reg; 110 111 r->val = skx_readx(r->vbase + r->offset, r->width); 112 } 113 114 /* Read a local-view register. */ 115 static bool read_local_reg(struct local_reg *reg) 116 { 117 int cpu; 118 119 /* Get the target CPU in the package @reg->pkg. */ 120 for_each_online_cpu(cpu) { 121 if (reg->pkg == topology_physical_package_id(cpu)) 122 break; 123 } 124 125 if (cpu >= nr_cpu_ids) 126 return false; 127 128 reg->vbase = ioremap(reg->pbase, reg->size); 129 if (!reg->vbase) { 130 imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", reg->pbase); 131 return false; 132 } 133 134 /* Get the target CPU to read the register. */ 135 smp_call_function_single(cpu, __read_local_reg, reg, 1); 136 iounmap(reg->vbase); 137 138 return true; 139 } 140 141 /* Get the bitmap of memory controller instances in package @pkg. */ 142 static u32 get_imc_bitmap(struct res_config *cfg, int pkg, bool north) 143 { 144 DEFINE_LOCAL_REG(reg, cfg, pkg, north, pcu, 0, capid3); 145 146 if (!read_local_reg(®)) 147 return 0; 148 149 edac_dbg(2, "Pkg%d %s mc instances bitmap 0x%llx (reg 0x%llx)\n", 150 pkg, north ? "north" : "south", 151 DDR_IMC_BITMAP(reg.val), reg.val); 152 153 return DDR_IMC_BITMAP(reg.val); 154 } 155 156 static void imc_release(struct device *dev) 157 { 158 edac_dbg(2, "imc device %s released\n", dev_name(dev)); 159 kfree(dev); 160 } 161 162 static int __get_ddr_munits(struct res_config *cfg, struct skx_dev *d, 163 bool north, int lmc) 164 { 165 unsigned long size = cfg->ddr_chan_mmio_sz * cfg->ddr_chan_num; 166 unsigned long bitmap = get_imc_bitmap(cfg, d->pkg, north); 167 void __iomem *mbase; 168 struct device *dev; 169 int i, rc, pmc; 170 u64 base; 171 172 for_each_set_bit(i, &bitmap, sizeof(bitmap) * 8) { 173 base = north ? d->mmio_base_h_north : d->mmio_base_h_south; 174 base += cfg->ddr_imc_base + size * i; 175 176 edac_dbg(2, "Pkg%d mc%d mmio base 0x%llx size 0x%lx\n", 177 d->pkg, lmc, base, size); 178 179 /* Set up the imc MMIO. */ 180 mbase = ioremap(base, size); 181 if (!mbase) { 182 imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", base); 183 return -ENOMEM; 184 } 185 186 d->imc[lmc].mbase = mbase; 187 d->imc[lmc].lmc = lmc; 188 189 /* Create the imc device instance. */ 190 dev = kzalloc_obj(*dev); 191 if (!dev) 192 return -ENOMEM; 193 194 dev->release = imc_release; 195 device_initialize(dev); 196 rc = dev_set_name(dev, "0x%llx", base); 197 if (rc) { 198 imh_printk(KERN_ERR, "Failed to set dev name\n"); 199 put_device(dev); 200 return rc; 201 } 202 203 d->imc[lmc].dev = dev; 204 205 /* Set up the imc index mapping. */ 206 pmc = north ? i : 8 + i; 207 skx_set_mc_mapping(d, pmc, lmc); 208 209 lmc++; 210 } 211 212 return lmc; 213 } 214 215 static bool get_ddr_munits(struct res_config *cfg, struct skx_dev *d) 216 { 217 int lmc = __get_ddr_munits(cfg, d, true, 0); 218 219 if (lmc < 0) 220 return false; 221 222 lmc = __get_ddr_munits(cfg, d, false, lmc); 223 if (lmc <= 0) 224 return false; 225 226 return true; 227 } 228 229 static bool get_socket_id(struct res_config *cfg, struct skx_dev *d) 230 { 231 DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ubox, 0, socket_id); 232 u8 src_id; 233 int i; 234 235 if (!read_local_reg(®)) 236 return false; 237 238 src_id = SOCKET_ID(reg.val); 239 edac_dbg(2, "socket id 0x%x (reg 0x%llx)\n", src_id, reg.val); 240 241 for (i = 0; i < cfg->ddr_imc_num; i++) 242 d->imc[i].src_id = src_id; 243 244 return true; 245 } 246 247 /* Get TOLM (Top Of Low Memory) and TOHM (Top Of High Memory) parameters. */ 248 static bool imh_get_tolm_tohm(struct res_config *cfg, u64 *tolm, u64 *tohm) 249 { 250 DEFINE_LOCAL_REG(reg, cfg, 0, true, sca, 0, tolm); 251 252 if (!read_local_reg(®)) 253 return false; 254 255 *tolm = TOLM(reg.val); 256 edac_dbg(2, "tolm 0x%llx (reg 0x%llx)\n", *tolm, reg.val); 257 258 DEFINE_LOCAL_REG(reg2, cfg, 0, true, sca, 0, tohm); 259 260 if (!read_local_reg(®2)) 261 return false; 262 263 *tohm = TOHM(reg2.val); 264 edac_dbg(2, "tohm 0x%llx (reg 0x%llx)\n", *tohm, reg2.val); 265 266 return true; 267 } 268 269 /* Get the system-view MMIO_BASE_H for {north,south}-IMH. */ 270 static int imh_get_all_mmio_base_h(struct res_config *cfg, struct list_head *edac_list) 271 { 272 int i, n = topology_max_packages(), imc_num = cfg->ddr_imc_num + cfg->hbm_imc_num; 273 struct skx_dev *d; 274 275 for (i = 0; i < n; i++) { 276 d = kzalloc_flex(*d, imc, imc_num); 277 if (!d) 278 return -ENOMEM; 279 280 DEFINE_LOCAL_REG(reg, cfg, i, true, ubox, 0, mmio_base); 281 282 /* Get MMIO_BASE_H for the north-IMH. */ 283 if (!read_local_reg(®) || !reg.val) { 284 kfree(d); 285 imh_printk(KERN_ERR, "Pkg%d has no north mmio_base_h\n", i); 286 return -ENODEV; 287 } 288 289 d->mmio_base_h_north = MMIO_BASE_H(reg.val); 290 edac_dbg(2, "Pkg%d north mmio_base_h 0x%llx (reg 0x%llx)\n", 291 i, d->mmio_base_h_north, reg.val); 292 293 /* Get MMIO_BASE_H for the south-IMH (optional). */ 294 DEFINE_LOCAL_REG(reg2, cfg, i, false, ubox, 0, mmio_base); 295 296 if (read_local_reg(®2)) { 297 d->mmio_base_h_south = MMIO_BASE_H(reg2.val); 298 edac_dbg(2, "Pkg%d south mmio_base_h 0x%llx (reg 0x%llx)\n", 299 i, d->mmio_base_h_south, reg2.val); 300 } 301 302 d->pkg = i; 303 d->num_imc = imc_num; 304 skx_init_mc_mapping(d); 305 list_add_tail(&d->list, edac_list); 306 } 307 308 return 0; 309 } 310 311 /* Get the number of per-package memory controllers. */ 312 static int imh_get_imc_num(struct res_config *cfg) 313 { 314 int imc_num = hweight32(get_imc_bitmap(cfg, 0, true)) + 315 hweight32(get_imc_bitmap(cfg, 0, false)); 316 317 if (!imc_num) { 318 imh_printk(KERN_ERR, "Invalid mc number\n"); 319 return -ENODEV; 320 } 321 322 if (cfg->ddr_imc_num != imc_num) { 323 /* 324 * Update the configuration data to reflect the number of 325 * present DDR memory controllers. 326 */ 327 cfg->ddr_imc_num = imc_num; 328 edac_dbg(2, "Set ddr mc number %d\n", imc_num); 329 } 330 331 return 0; 332 } 333 334 /* Get all memory controllers' parameters. */ 335 static int imh_get_munits(struct res_config *cfg, struct list_head *edac_list) 336 { 337 struct skx_imc *imc; 338 struct skx_dev *d; 339 u8 mc = 0; 340 int i; 341 342 list_for_each_entry(d, edac_list, list) { 343 if (!get_ddr_munits(cfg, d)) { 344 imh_printk(KERN_ERR, "No mc found\n"); 345 return -ENODEV; 346 } 347 348 if (!get_socket_id(cfg, d)) { 349 imh_printk(KERN_ERR, "Failed to get socket id\n"); 350 return -ENODEV; 351 } 352 353 for (i = 0; i < cfg->ddr_imc_num; i++) { 354 imc = &d->imc[i]; 355 if (!imc->mbase) 356 continue; 357 358 imc->chan_mmio_sz = cfg->ddr_chan_mmio_sz; 359 imc->num_channels = cfg->ddr_chan_num; 360 imc->num_dimms = cfg->ddr_dimm_num; 361 imc->mc = mc++; 362 } 363 } 364 365 return 0; 366 } 367 368 static bool check_2lm_enabled(struct res_config *cfg, struct skx_dev *d, int ha_idx) 369 { 370 DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ha, ha_idx, mode); 371 372 if (!read_local_reg(®)) 373 return false; 374 375 if (!NMCACHING(reg.val)) 376 return false; 377 378 edac_dbg(2, "2-level memory configuration (reg 0x%llx, ha idx %d)\n", reg.val, ha_idx); 379 return true; 380 } 381 382 /* Check whether the system has a 2-level memory configuration. */ 383 static bool imh_2lm_enabled(struct res_config *cfg, struct list_head *head) 384 { 385 struct skx_dev *d; 386 int i; 387 388 list_for_each_entry(d, head, list) { 389 for (i = 0; i < cfg->ddr_imc_num; i++) 390 if (check_2lm_enabled(cfg, d, i)) 391 return true; 392 } 393 394 return false; 395 } 396 397 static u32 read_imc_mcmtr(struct res_config *cfg, struct skx_imc *imc, int chan) 398 { 399 return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_mcmtr_offset, cfg->ddr_reg_mcmtr_width); 400 } 401 402 static u32 read_imc_dimmmtr(struct res_config *cfg, struct skx_imc *imc, int chan, int dimm) 403 { 404 return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_dimmmtr_offset + 405 cfg->ddr_reg_dimmmtr_width * dimm, 406 cfg->ddr_reg_dimmmtr_width); 407 } 408 409 static bool ecc_enabled(u32 mcmtr) 410 { 411 return (bool)ECC_ENABLED(mcmtr); 412 } 413 414 static bool dimm_populated(u32 dimmmtr) 415 { 416 return (bool)DIMM_POPULATED(dimmmtr); 417 } 418 419 /* Get each DIMM's configurations of the memory controller @mci. */ 420 static int imh_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg) 421 { 422 struct skx_pvt *pvt = mci->pvt_info; 423 struct skx_imc *imc = pvt->imc; 424 struct dimm_info *dimm; 425 u32 mcmtr, dimmmtr; 426 int i, j, ndimms; 427 428 for (i = 0; i < imc->num_channels; i++) { 429 if (!imc->mbase) 430 continue; 431 432 mcmtr = read_imc_mcmtr(cfg, imc, i); 433 434 for (ndimms = 0, j = 0; j < imc->num_dimms; j++) { 435 dimmmtr = read_imc_dimmmtr(cfg, imc, i, j); 436 edac_dbg(1, "mcmtr 0x%x dimmmtr 0x%x (mc%d ch%d dimm%d)\n", 437 mcmtr, dimmmtr, imc->mc, i, j); 438 439 if (!dimm_populated(dimmmtr)) 440 continue; 441 442 dimm = edac_get_dimm(mci, i, j, 0); 443 ndimms += skx_get_dimm_info(dimmmtr, 0, 0, dimm, 444 imc, i, j, cfg); 445 } 446 447 if (ndimms && !ecc_enabled(mcmtr)) { 448 imh_printk(KERN_ERR, "ECC is disabled on mc%d ch%d\n", 449 imc->mc, i); 450 return -ENODEV; 451 } 452 } 453 454 return 0; 455 } 456 457 /* Register all memory controllers to the EDAC core. */ 458 static int imh_register_mci(struct res_config *cfg, struct list_head *edac_list) 459 { 460 struct skx_imc *imc; 461 struct skx_dev *d; 462 int i, rc; 463 464 list_for_each_entry(d, edac_list, list) { 465 for (i = 0; i < cfg->ddr_imc_num; i++) { 466 imc = &d->imc[i]; 467 if (!imc->mbase) 468 continue; 469 470 rc = skx_register_mci(imc, imc->dev, 471 dev_name(imc->dev), 472 "Intel IMH-based Socket", 473 EDAC_MOD_STR, 474 imh_get_dimm_config, cfg); 475 if (rc) 476 return rc; 477 } 478 } 479 480 return 0; 481 } 482 483 static struct res_config dmr_cfg = { 484 .type = DMR, 485 .support_ddr5 = true, 486 .mmio_base_l_north = 0xf6800000, 487 .mmio_base_l_south = 0xf6000000, 488 .ddr_chan_num = 1, 489 .ddr_dimm_num = 2, 490 .ddr_imc_base = 0x39b000, 491 .ddr_chan_mmio_sz = 0x8000, 492 .ddr_reg_mcmtr_offset = 0x360, 493 .ddr_reg_mcmtr_width = 4, 494 .ddr_reg_dimmmtr_offset = 0x370, 495 .ddr_reg_dimmmtr_width = 4, 496 .ubox_base = 0x0, 497 .ubox_size = 0x2000, 498 .ubox_reg_mmio_base_offset = 0x580, 499 .ubox_reg_mmio_base_width = 4, 500 .ubox_reg_socket_id_offset = 0x1080, 501 .ubox_reg_socket_id_width = 4, 502 .pcu_base = 0x3000, 503 .pcu_size = 0x10000, 504 .pcu_reg_capid3_offset = 0x290, 505 .pcu_reg_capid3_width = 4, 506 .sca_base = 0x24c000, 507 .sca_size = 0x2500, 508 .sca_reg_tolm_offset = 0x2100, 509 .sca_reg_tolm_width = 8, 510 .sca_reg_tohm_offset = 0x2108, 511 .sca_reg_tohm_width = 8, 512 .ha_base = 0x3eb000, 513 .ha_size = 0x1000, 514 .ha_reg_mode_offset = 0x4a0, 515 .ha_reg_mode_width = 4, 516 .reg_rrl_ddr[0] = &dmr_reg_rrl_ddr_subch0, 517 .reg_rrl_ddr[1] = &dmr_reg_rrl_ddr_subch1, 518 }; 519 520 static const struct x86_cpu_id imh_cpuids[] = { 521 X86_MATCH_VFM(INTEL_DIAMONDRAPIDS_X, &dmr_cfg), 522 {} 523 }; 524 MODULE_DEVICE_TABLE(x86cpu, imh_cpuids); 525 526 static struct notifier_block imh_mce_dec = { 527 .notifier_call = skx_mce_check_error, 528 .priority = MCE_PRIO_EDAC, 529 }; 530 531 static int __init imh_init(void) 532 { 533 const struct x86_cpu_id *id; 534 struct list_head *edac_list; 535 struct res_config *cfg; 536 const char *owner; 537 u64 tolm, tohm; 538 int rc; 539 540 edac_dbg(2, "\n"); 541 542 if (ghes_get_devices()) 543 return -EBUSY; 544 545 owner = edac_get_owner(); 546 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 547 return -EBUSY; 548 549 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 550 return -ENODEV; 551 552 id = x86_match_cpu(imh_cpuids); 553 if (!id) 554 return -ENODEV; 555 cfg = (struct res_config *)id->driver_data; 556 skx_set_res_cfg(cfg); 557 res_cfg = cfg; 558 559 if (!imh_get_tolm_tohm(cfg, &tolm, &tohm)) 560 return -ENODEV; 561 562 skx_set_hi_lo(tolm, tohm); 563 564 rc = imh_get_imc_num(cfg); 565 if (rc < 0) 566 goto fail; 567 568 edac_list = skx_get_edac_list(); 569 570 rc = imh_get_all_mmio_base_h(cfg, edac_list); 571 if (rc) 572 goto fail; 573 574 rc = imh_get_munits(cfg, edac_list); 575 if (rc) 576 goto fail; 577 578 skx_set_mem_cfg(imh_2lm_enabled(cfg, edac_list)); 579 580 rc = imh_register_mci(cfg, edac_list); 581 if (rc) 582 goto fail; 583 584 rc = skx_adxl_get(); 585 if (rc) 586 goto fail; 587 588 opstate_init(); 589 mce_register_decode_chain(&imh_mce_dec); 590 skx_setup_debug("imh_test"); 591 592 cfg->rrl_ctrl_mode = retry_rd_err_log; 593 if (retry_rd_err_log && cfg->reg_rrl_ddr[0]) { 594 skx_set_show_rrl(skx_show_rrl); 595 if (retry_rd_err_log == RRL_CTRL_LINUX) 596 skx_enable_rrl(true); 597 } 598 599 imh_printk(KERN_INFO, "%s\n", IMH_REVISION); 600 601 return 0; 602 fail: 603 skx_remove(); 604 return rc; 605 } 606 607 static void __exit imh_exit(void) 608 { 609 edac_dbg(2, "\n"); 610 611 if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) { 612 if (retry_rd_err_log == RRL_CTRL_LINUX) 613 skx_enable_rrl(false); 614 skx_set_show_rrl(NULL); 615 } 616 617 skx_teardown_debug(); 618 mce_unregister_decode_chain(&imh_mce_dec); 619 skx_adxl_put(); 620 skx_remove(); 621 } 622 623 module_init(imh_init); 624 module_exit(imh_exit); 625 626 module_param(retry_rd_err_log, int, 0444); 627 MODULE_PARM_DESC(retry_rd_err_log, "retry_rd_err_log: 0=off(default), 1=bios(Linux doesn't reset any control bits, but just reports values.), 2=linux(Linux tries to take control and resets mode bits, clear valid/UC bits after reading.)"); 628 629 MODULE_LICENSE("GPL"); 630 MODULE_AUTHOR("Qiuxu Zhuo"); 631 MODULE_DESCRIPTION("MC Driver for Intel servers using IMH-based memory controller"); 632