1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Driver for Intel(R) 10nm server memory controller. 4 * Copyright (c) 2019, Intel Corporation. 5 * 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/io.h> 10 #include <asm/cpu_device_id.h> 11 #include <asm/intel-family.h> 12 #include <asm/mce.h> 13 #include "edac_module.h" 14 #include "skx_common.h" 15 16 #define I10NM_REVISION "v0.0.6" 17 #define EDAC_MOD_STR "i10nm_edac" 18 19 /* Debug macros */ 20 #define i10nm_printk(level, fmt, arg...) \ 21 edac_printk(level, "i10nm", fmt, ##arg) 22 23 #define I10NM_GET_SCK_BAR(d, reg) \ 24 pci_read_config_dword((d)->uracu, 0xd0, &(reg)) 25 #define I10NM_GET_IMC_BAR(d, i, reg) \ 26 pci_read_config_dword((d)->uracu, \ 27 (res_cfg->type == GNR ? 0xd4 : 0xd8) + (i) * 4, &(reg)) 28 #define I10NM_GET_SAD(d, offset, i, reg)\ 29 pci_read_config_dword((d)->sad_all, (offset) + (i) * \ 30 (res_cfg->type == GNR ? 12 : 8), &(reg)) 31 #define I10NM_GET_HBM_IMC_BAR(d, reg) \ 32 pci_read_config_dword((d)->uracu, 0xd4, &(reg)) 33 #define I10NM_GET_CAPID3_CFG(d, reg) \ 34 pci_read_config_dword((d)->pcu_cr3, \ 35 res_cfg->type == GNR ? 0x290 : 0x90, &(reg)) 36 #define I10NM_GET_CAPID5_CFG(d, reg) \ 37 pci_read_config_dword((d)->pcu_cr3, \ 38 res_cfg->type == GNR ? 0x298 : 0x98, &(reg)) 39 #define I10NM_GET_DIMMMTR(m, i, j) \ 40 readl((m)->mbase + ((m)->hbm_mc ? 0x80c : \ 41 (res_cfg->type == GNR ? 0xc0c : 0x2080c)) + \ 42 (i) * (m)->chan_mmio_sz + (j) * 4) 43 #define I10NM_GET_MCDDRTCFG(m, i) \ 44 readl((m)->mbase + ((m)->hbm_mc ? 0x970 : 0x20970) + \ 45 (i) * (m)->chan_mmio_sz) 46 #define I10NM_GET_MCMTR(m, i) \ 47 readl((m)->mbase + ((m)->hbm_mc ? 0xef8 : \ 48 (res_cfg->type == GNR ? 0xaf8 : 0x20ef8)) + \ 49 (i) * (m)->chan_mmio_sz) 50 51 #define I10NM_GET_SCK_MMIO_BASE(reg) (GET_BITFIELD(reg, 0, 28) << 23) 52 #define I10NM_GET_IMC_MMIO_OFFSET(reg) (GET_BITFIELD(reg, 0, 10) << 12) 53 #define I10NM_GET_IMC_MMIO_SIZE(reg) ((GET_BITFIELD(reg, 13, 23) - \ 54 GET_BITFIELD(reg, 0, 10) + 1) << 12) 55 #define I10NM_GET_HBM_IMC_MMIO_OFFSET(reg) \ 56 ((GET_BITFIELD(reg, 0, 10) << 12) + 0x140000) 57 58 #define I10NM_GNR_IMC_MMIO_OFFSET 0x24c000 59 #define I10NM_GNR_D_IMC_MMIO_OFFSET 0x206000 60 #define I10NM_GNR_IMC_MMIO_SIZE 0x4000 61 #define I10NM_HBM_IMC_MMIO_SIZE 0x9000 62 #define I10NM_DDR_IMC_CH_CNT(reg) GET_BITFIELD(reg, 21, 24) 63 #define I10NM_IS_HBM_PRESENT(reg) GET_BITFIELD(reg, 27, 30) 64 #define I10NM_IS_HBM_IMC(reg) GET_BITFIELD(reg, 29, 29) 65 66 #define I10NM_MAX_SAD 16 67 #define I10NM_SAD_ENABLE(reg) GET_BITFIELD(reg, 0, 0) 68 #define I10NM_SAD_NM_CACHEABLE(reg) GET_BITFIELD(reg, 5, 5) 69 70 static struct list_head *i10nm_edac_list; 71 72 static struct res_config *res_cfg; 73 static int retry_rd_err_log; 74 static int decoding_via_mca; 75 static bool mem_cfg_2lm; 76 static bool no_adxl; 77 78 static struct reg_rrl icx_reg_rrl_ddr = { 79 .set_num = 2, 80 .reg_num = 6, 81 .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, 82 .offsets = { 83 {0x22c60, 0x22c54, 0x22c5c, 0x22c58, 0x22c28, 0x20ed8}, 84 {0x22e54, 0x22e60, 0x22e64, 0x22e58, 0x22e5c, 0x20ee0}, 85 }, 86 .widths = {4, 4, 4, 4, 4, 8}, 87 .v_mask = BIT(0), 88 .uc_mask = BIT(1), 89 .over_mask = BIT(2), 90 .en_patspr_mask = BIT(13), 91 .noover_mask = BIT(14), 92 .en_mask = BIT(15), 93 94 .cecnt_num = 4, 95 .cecnt_offsets = {0x22c18, 0x22c1c, 0x22c20, 0x22c24}, 96 .cecnt_widths = {4, 4, 4, 4}, 97 }; 98 99 static struct reg_rrl spr_reg_rrl_ddr = { 100 .set_num = 3, 101 .reg_num = 6, 102 .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND, RRL_SRC_FRE_DEMAND}, 103 .offsets = { 104 {0x22c60, 0x22c54, 0x22f08, 0x22c58, 0x22c28, 0x20ed8}, 105 {0x22e54, 0x22e60, 0x22f10, 0x22e58, 0x22e5c, 0x20ee0}, 106 {0x22c70, 0x22d80, 0x22f18, 0x22d58, 0x22c64, 0x20f10}, 107 }, 108 .widths = {4, 4, 8, 4, 4, 8}, 109 .v_mask = BIT(0), 110 .uc_mask = BIT(1), 111 .over_mask = BIT(2), 112 .en_patspr_mask = BIT(13), 113 .noover_mask = BIT(14), 114 .en_mask = BIT(15), 115 116 .cecnt_num = 4, 117 .cecnt_offsets = {0x22c18, 0x22c1c, 0x22c20, 0x22c24}, 118 .cecnt_widths = {4, 4, 4, 4}, 119 }; 120 121 static struct reg_rrl spr_reg_rrl_hbm_pch0 = { 122 .set_num = 2, 123 .reg_num = 6, 124 .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, 125 .offsets = { 126 {0x2860, 0x2854, 0x2b08, 0x2858, 0x2828, 0x0ed8}, 127 {0x2a54, 0x2a60, 0x2b10, 0x2a58, 0x2a5c, 0x0ee0}, 128 }, 129 .widths = {4, 4, 8, 4, 4, 8}, 130 .v_mask = BIT(0), 131 .uc_mask = BIT(1), 132 .over_mask = BIT(2), 133 .en_patspr_mask = BIT(13), 134 .noover_mask = BIT(14), 135 .en_mask = BIT(15), 136 137 .cecnt_num = 4, 138 .cecnt_offsets = {0x2818, 0x281c, 0x2820, 0x2824}, 139 .cecnt_widths = {4, 4, 4, 4}, 140 }; 141 142 static struct reg_rrl spr_reg_rrl_hbm_pch1 = { 143 .set_num = 2, 144 .reg_num = 6, 145 .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, 146 .offsets = { 147 {0x2c60, 0x2c54, 0x2f08, 0x2c58, 0x2c28, 0x0fa8}, 148 {0x2e54, 0x2e60, 0x2f10, 0x2e58, 0x2e5c, 0x0fb0}, 149 }, 150 .widths = {4, 4, 8, 4, 4, 8}, 151 .v_mask = BIT(0), 152 .uc_mask = BIT(1), 153 .over_mask = BIT(2), 154 .en_patspr_mask = BIT(13), 155 .noover_mask = BIT(14), 156 .en_mask = BIT(15), 157 158 .cecnt_num = 4, 159 .cecnt_offsets = {0x2c18, 0x2c1c, 0x2c20, 0x2c24}, 160 .cecnt_widths = {4, 4, 4, 4}, 161 }; 162 163 static struct reg_rrl gnr_reg_rrl_ddr = { 164 .set_num = 4, 165 .reg_num = 6, 166 .sources = {RRL_SRC_FRE_SCRUB, RRL_SRC_FRE_DEMAND, RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, 167 .offsets = { 168 {0x2f10, 0x2f20, 0x2f30, 0x2f50, 0x2f60, 0xba0}, 169 {0x2f14, 0x2f24, 0x2f38, 0x2f54, 0x2f64, 0xba8}, 170 {0x2f18, 0x2f28, 0x2f40, 0x2f58, 0x2f68, 0xbb0}, 171 {0x2f1c, 0x2f2c, 0x2f48, 0x2f5c, 0x2f6c, 0xbb8}, 172 }, 173 .widths = {4, 4, 8, 4, 4, 8}, 174 .v_mask = BIT(0), 175 .uc_mask = BIT(1), 176 .over_mask = BIT(2), 177 .en_patspr_mask = BIT(14), 178 .noover_mask = BIT(15), 179 .en_mask = BIT(12), 180 181 .cecnt_num = 8, 182 .cecnt_offsets = {0x2c10, 0x2c14, 0x2c18, 0x2c1c, 0x2c20, 0x2c24, 0x2c28, 0x2c2c}, 183 .cecnt_widths = {4, 4, 4, 4, 4, 4, 4, 4}, 184 }; 185 186 static struct pci_dev *pci_get_dev_wrapper(int dom, unsigned int bus, 187 unsigned int dev, unsigned int fun) 188 { 189 struct pci_dev *pdev; 190 191 pdev = pci_get_domain_bus_and_slot(dom, bus, PCI_DEVFN(dev, fun)); 192 if (!pdev) { 193 edac_dbg(2, "No device %02x:%02x.%x\n", 194 bus, dev, fun); 195 return NULL; 196 } 197 198 if (unlikely(pci_enable_device(pdev) < 0)) { 199 edac_dbg(2, "Failed to enable device %02x:%02x.%x\n", 200 bus, dev, fun); 201 pci_dev_put(pdev); 202 return NULL; 203 } 204 205 return pdev; 206 } 207 208 /** 209 * i10nm_get_imc_num() - Get the number of present DDR memory controllers. 210 * 211 * @cfg : The pointer to the structure of EDAC resource configurations. 212 * 213 * For Granite Rapids CPUs, the number of present DDR memory controllers read 214 * at runtime overwrites the value statically configured in @cfg->ddr_imc_num. 215 * For other CPUs, the number of present DDR memory controllers is statically 216 * configured in @cfg->ddr_imc_num. 217 * 218 * RETURNS : 0 on success, < 0 on failure. 219 */ 220 static int i10nm_get_imc_num(struct res_config *cfg) 221 { 222 int n, imc_num, chan_num = 0; 223 struct skx_dev *d; 224 u32 reg; 225 226 list_for_each_entry(d, i10nm_edac_list, list) { 227 d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->pcu_cr3_bdf.bus], 228 res_cfg->pcu_cr3_bdf.dev, 229 res_cfg->pcu_cr3_bdf.fun); 230 if (!d->pcu_cr3) 231 continue; 232 233 if (I10NM_GET_CAPID5_CFG(d, reg)) 234 continue; 235 236 n = I10NM_DDR_IMC_CH_CNT(reg); 237 238 if (!chan_num) { 239 chan_num = n; 240 edac_dbg(2, "Get DDR CH number: %d\n", chan_num); 241 } else if (chan_num != n) { 242 i10nm_printk(KERN_NOTICE, "Get DDR CH numbers: %d, %d\n", chan_num, n); 243 } 244 } 245 246 switch (cfg->type) { 247 case GNR: 248 /* 249 * One channel per DDR memory controller for Granite Rapids CPUs. 250 */ 251 imc_num = chan_num; 252 253 if (!imc_num) { 254 i10nm_printk(KERN_ERR, "Invalid DDR MC number\n"); 255 return -ENODEV; 256 } 257 258 if (cfg->ddr_imc_num != imc_num) { 259 /* 260 * Update the configuration data to reflect the number of 261 * present DDR memory controllers. 262 */ 263 cfg->ddr_imc_num = imc_num; 264 edac_dbg(2, "Set DDR MC number: %d", imc_num); 265 266 /* Release and reallocate skx_dev list with the updated number. */ 267 skx_remove(); 268 if (skx_get_all_bus_mappings(cfg, &i10nm_edac_list) <= 0) 269 return -ENODEV; 270 } 271 272 return 0; 273 default: 274 /* 275 * For other CPUs, the number of present DDR memory controllers 276 * is statically pre-configured in cfg->ddr_imc_num. 277 */ 278 return 0; 279 } 280 } 281 282 static bool i10nm_check_2lm(struct res_config *cfg) 283 { 284 struct skx_dev *d; 285 u32 reg; 286 int i; 287 288 list_for_each_entry(d, i10nm_edac_list, list) { 289 d->sad_all = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->sad_all_bdf.bus], 290 res_cfg->sad_all_bdf.dev, 291 res_cfg->sad_all_bdf.fun); 292 if (!d->sad_all) 293 continue; 294 295 for (i = 0; i < I10NM_MAX_SAD; i++) { 296 I10NM_GET_SAD(d, cfg->sad_all_offset, i, reg); 297 if (I10NM_SAD_ENABLE(reg) && I10NM_SAD_NM_CACHEABLE(reg)) { 298 edac_dbg(2, "2-level memory configuration.\n"); 299 return true; 300 } 301 } 302 } 303 304 return false; 305 } 306 307 /* 308 * Check whether the error comes from DDRT by ICX/Tremont/SPR model specific error code. 309 * Refer to SDM vol3B 17.11.3/17.13.2 Intel IMC MC error codes for IA32_MCi_STATUS. 310 */ 311 static bool i10nm_mscod_is_ddrt(u32 mscod) 312 { 313 switch (res_cfg->type) { 314 case I10NM: 315 switch (mscod) { 316 case 0x0106: case 0x0107: 317 case 0x0800: case 0x0804: 318 case 0x0806 ... 0x0808: 319 case 0x080a ... 0x080e: 320 case 0x0810: case 0x0811: 321 case 0x0816: case 0x081e: 322 case 0x081f: 323 return true; 324 } 325 326 break; 327 case SPR: 328 switch (mscod) { 329 case 0x0800: case 0x0804: 330 case 0x0806 ... 0x0808: 331 case 0x080a ... 0x080e: 332 case 0x0810: case 0x0811: 333 case 0x0816: case 0x081e: 334 case 0x081f: 335 return true; 336 } 337 338 break; 339 default: 340 return false; 341 } 342 343 return false; 344 } 345 346 static bool i10nm_mc_decode_available(struct mce *mce) 347 { 348 #define ICX_IMCx_CHy 0x06666000 349 u8 bank; 350 351 if (!decoding_via_mca || mem_cfg_2lm) 352 return false; 353 354 if ((mce->status & (MCI_STATUS_MISCV | MCI_STATUS_ADDRV)) 355 != (MCI_STATUS_MISCV | MCI_STATUS_ADDRV)) 356 return false; 357 358 bank = mce->bank; 359 360 switch (res_cfg->type) { 361 case I10NM: 362 /* Check whether the bank is one of {13,14,17,18,21,22,25,26} */ 363 if (!(ICX_IMCx_CHy & (1 << bank))) 364 return false; 365 break; 366 case SPR: 367 if (bank < 13 || bank > 20) 368 return false; 369 break; 370 case GNR: 371 if (bank < 13 || bank > 24) 372 return false; 373 break; 374 default: 375 return false; 376 } 377 378 /* DDRT errors can't be decoded from MCA bank registers */ 379 if (MCI_MISC_ECC_MODE(mce->misc) == MCI_MISC_ECC_DDRT) 380 return false; 381 382 if (i10nm_mscod_is_ddrt(MCI_STATUS_MSCOD(mce->status))) 383 return false; 384 385 return true; 386 } 387 388 static bool i10nm_mc_decode(struct decoded_addr *res) 389 { 390 struct mce *m = res->mce; 391 struct skx_dev *d; 392 u8 bank; 393 394 if (!i10nm_mc_decode_available(m)) 395 return false; 396 397 list_for_each_entry(d, i10nm_edac_list, list) { 398 if (d->imc[0].src_id == m->socketid) { 399 res->socket = m->socketid; 400 res->dev = d; 401 break; 402 } 403 } 404 405 switch (res_cfg->type) { 406 case I10NM: 407 bank = m->bank - 13; 408 res->imc = bank / 4; 409 res->channel = bank % 2; 410 res->column = GET_BITFIELD(m->misc, 9, 18) << 2; 411 res->row = GET_BITFIELD(m->misc, 19, 39); 412 res->bank_group = GET_BITFIELD(m->misc, 40, 41); 413 res->bank_address = GET_BITFIELD(m->misc, 42, 43); 414 res->bank_group |= GET_BITFIELD(m->misc, 44, 44) << 2; 415 res->rank = GET_BITFIELD(m->misc, 56, 58); 416 res->dimm = res->rank >> 2; 417 res->rank = res->rank % 4; 418 break; 419 case SPR: 420 bank = m->bank - 13; 421 res->imc = bank / 2; 422 res->channel = bank % 2; 423 res->column = GET_BITFIELD(m->misc, 9, 18) << 2; 424 res->row = GET_BITFIELD(m->misc, 19, 36); 425 res->bank_group = GET_BITFIELD(m->misc, 37, 38); 426 res->bank_address = GET_BITFIELD(m->misc, 39, 40); 427 res->bank_group |= GET_BITFIELD(m->misc, 41, 41) << 2; 428 res->rank = GET_BITFIELD(m->misc, 57, 57); 429 res->dimm = GET_BITFIELD(m->misc, 58, 58); 430 break; 431 case GNR: 432 res->imc = m->bank - 13; 433 res->channel = 0; 434 res->column = GET_BITFIELD(m->misc, 9, 18) << 2; 435 res->row = GET_BITFIELD(m->misc, 19, 36); 436 res->bank_group = GET_BITFIELD(m->misc, 39, 41); 437 res->bank_address = GET_BITFIELD(m->misc, 37, 38); 438 res->rank = GET_BITFIELD(m->misc, 55, 56); 439 res->dimm = GET_BITFIELD(m->misc, 57, 57); 440 break; 441 default: 442 return false; 443 } 444 445 if (!res->dev) { 446 skx_printk(KERN_ERR, "No device for src_id %d imc %d\n", 447 m->socketid, res->imc); 448 return false; 449 } 450 451 return true; 452 } 453 454 /** 455 * get_gnr_mdev() - Get the PCI device of the @logical_idx-th DDR memory controller. 456 * 457 * @d : The pointer to the structure of CPU socket EDAC device. 458 * @logical_idx : The logical index of the present memory controller (0 ~ max present MC# - 1). 459 * @physical_idx : To store the corresponding physical index of @logical_idx. 460 * 461 * RETURNS : The PCI device of the @logical_idx-th DDR memory controller, NULL on failure. 462 */ 463 static struct pci_dev *get_gnr_mdev(struct skx_dev *d, int logical_idx, int *physical_idx) 464 { 465 #define GNR_MAX_IMC_PCI_CNT 28 466 467 struct pci_dev *mdev; 468 int i, logical = 0; 469 470 /* 471 * Detect present memory controllers from { PCI device: 8-5, function 7-1 } 472 */ 473 for (i = 0; i < GNR_MAX_IMC_PCI_CNT; i++) { 474 mdev = pci_get_dev_wrapper(d->seg, 475 d->bus[res_cfg->ddr_mdev_bdf.bus], 476 res_cfg->ddr_mdev_bdf.dev + i / 7, 477 res_cfg->ddr_mdev_bdf.fun + i % 7); 478 479 if (mdev) { 480 if (logical == logical_idx) { 481 *physical_idx = i; 482 skx_set_mc_mapping(d, i, logical_idx); 483 return mdev; 484 } 485 486 pci_dev_put(mdev); 487 logical++; 488 } 489 } 490 491 return NULL; 492 } 493 494 static u32 get_gnr_imc_mmio_offset(void) 495 { 496 if (boot_cpu_data.x86_vfm == INTEL_GRANITERAPIDS_D) 497 return I10NM_GNR_D_IMC_MMIO_OFFSET; 498 499 return I10NM_GNR_IMC_MMIO_OFFSET; 500 } 501 502 /** 503 * get_ddr_munit() - Get the resource of the i-th DDR memory controller. 504 * 505 * @d : The pointer to the structure of CPU socket EDAC device. 506 * @i : The index of the CPU socket relative DDR memory controller. 507 * @offset : To store the MMIO offset of the i-th DDR memory controller. 508 * @size : To store the MMIO size of the i-th DDR memory controller. 509 * 510 * RETURNS : The PCI device of the i-th DDR memory controller, NULL on failure. 511 */ 512 static struct pci_dev *get_ddr_munit(struct skx_dev *d, int i, u32 *offset, unsigned long *size) 513 { 514 struct pci_dev *mdev; 515 int physical_idx; 516 u32 reg; 517 518 switch (res_cfg->type) { 519 case GNR: 520 if (I10NM_GET_IMC_BAR(d, 0, reg)) { 521 i10nm_printk(KERN_ERR, "Failed to get mc0 bar\n"); 522 return NULL; 523 } 524 525 mdev = get_gnr_mdev(d, i, &physical_idx); 526 if (!mdev) 527 return NULL; 528 529 *offset = I10NM_GET_IMC_MMIO_OFFSET(reg) + 530 get_gnr_imc_mmio_offset() + 531 physical_idx * I10NM_GNR_IMC_MMIO_SIZE; 532 *size = I10NM_GNR_IMC_MMIO_SIZE; 533 534 break; 535 default: 536 if (I10NM_GET_IMC_BAR(d, i, reg)) { 537 i10nm_printk(KERN_ERR, "Failed to get mc%d bar\n", i); 538 return NULL; 539 } 540 541 mdev = pci_get_dev_wrapper(d->seg, 542 d->bus[res_cfg->ddr_mdev_bdf.bus], 543 res_cfg->ddr_mdev_bdf.dev + i, 544 res_cfg->ddr_mdev_bdf.fun); 545 if (!mdev) 546 return NULL; 547 548 *offset = I10NM_GET_IMC_MMIO_OFFSET(reg); 549 *size = I10NM_GET_IMC_MMIO_SIZE(reg); 550 } 551 552 return mdev; 553 } 554 555 /** 556 * i10nm_imc_absent() - Check whether the memory controller @imc is absent 557 * 558 * @imc : The pointer to the structure of memory controller EDAC device. 559 * 560 * RETURNS : true if the memory controller EDAC device is absent, false otherwise. 561 */ 562 static bool i10nm_imc_absent(struct skx_imc *imc) 563 { 564 u32 mcmtr; 565 int i; 566 567 switch (res_cfg->type) { 568 case SPR: 569 for (i = 0; i < res_cfg->ddr_chan_num; i++) { 570 mcmtr = I10NM_GET_MCMTR(imc, i); 571 edac_dbg(1, "ch%d mcmtr reg %x\n", i, mcmtr); 572 if (mcmtr != ~0) 573 return false; 574 } 575 576 /* 577 * Some workstations' absent memory controllers still 578 * appear as PCIe devices, misleading the EDAC driver. 579 * By observing that the MMIO registers of these absent 580 * memory controllers consistently hold the value of ~0. 581 * 582 * We identify a memory controller as absent by checking 583 * if its MMIO register "mcmtr" == ~0 in all its channels. 584 */ 585 return true; 586 default: 587 return false; 588 } 589 } 590 591 static int i10nm_get_ddr_munits(void) 592 { 593 struct pci_dev *mdev; 594 void __iomem *mbase; 595 unsigned long size; 596 struct skx_dev *d; 597 int i, lmc, j = 0; 598 u32 reg, off; 599 u64 base; 600 601 list_for_each_entry(d, i10nm_edac_list, list) { 602 d->util_all = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->util_all_bdf.bus], 603 res_cfg->util_all_bdf.dev, 604 res_cfg->util_all_bdf.fun); 605 if (!d->util_all) 606 return -ENODEV; 607 608 d->uracu = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->uracu_bdf.bus], 609 res_cfg->uracu_bdf.dev, 610 res_cfg->uracu_bdf.fun); 611 if (!d->uracu) 612 return -ENODEV; 613 614 if (I10NM_GET_SCK_BAR(d, reg)) { 615 i10nm_printk(KERN_ERR, "Failed to socket bar\n"); 616 return -ENODEV; 617 } 618 619 base = I10NM_GET_SCK_MMIO_BASE(reg); 620 edac_dbg(2, "socket%d mmio base 0x%llx (reg 0x%x)\n", 621 j++, base, reg); 622 623 for (lmc = 0, i = 0; i < res_cfg->ddr_imc_num; i++) { 624 mdev = get_ddr_munit(d, i, &off, &size); 625 626 if (i == 0 && !mdev) { 627 i10nm_printk(KERN_ERR, "No IMC found\n"); 628 return -ENODEV; 629 } 630 if (!mdev) 631 continue; 632 633 edac_dbg(2, "mc%d mmio base 0x%llx size 0x%lx (reg 0x%x)\n", 634 i, base + off, size, reg); 635 636 mbase = ioremap(base + off, size); 637 if (!mbase) { 638 pci_dev_put(mdev); 639 i10nm_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", 640 base + off); 641 return -ENODEV; 642 } 643 644 d->imc[lmc].mbase = mbase; 645 if (i10nm_imc_absent(&d->imc[lmc])) { 646 pci_dev_put(mdev); 647 iounmap(mbase); 648 d->imc[lmc].mbase = NULL; 649 edac_dbg(2, "Skip absent mc%d\n", i); 650 continue; 651 } else { 652 d->imc[lmc].mdev = mdev; 653 if (res_cfg->type == SPR) 654 skx_set_mc_mapping(d, i, lmc); 655 lmc++; 656 } 657 } 658 } 659 660 return 0; 661 } 662 663 static bool i10nm_check_hbm_imc(struct skx_dev *d) 664 { 665 u32 reg; 666 667 if (I10NM_GET_CAPID3_CFG(d, reg)) { 668 i10nm_printk(KERN_ERR, "Failed to get capid3_cfg\n"); 669 return false; 670 } 671 672 return I10NM_IS_HBM_PRESENT(reg) != 0; 673 } 674 675 static int i10nm_get_hbm_munits(void) 676 { 677 struct pci_dev *mdev; 678 void __iomem *mbase; 679 u32 reg, off, mcmtr; 680 struct skx_dev *d; 681 int i, lmc; 682 u64 base; 683 684 list_for_each_entry(d, i10nm_edac_list, list) { 685 if (!d->pcu_cr3) 686 return -ENODEV; 687 688 if (!i10nm_check_hbm_imc(d)) { 689 i10nm_printk(KERN_DEBUG, "No hbm memory\n"); 690 return -ENODEV; 691 } 692 693 if (I10NM_GET_SCK_BAR(d, reg)) { 694 i10nm_printk(KERN_ERR, "Failed to get socket bar\n"); 695 return -ENODEV; 696 } 697 base = I10NM_GET_SCK_MMIO_BASE(reg); 698 699 if (I10NM_GET_HBM_IMC_BAR(d, reg)) { 700 i10nm_printk(KERN_ERR, "Failed to get hbm mc bar\n"); 701 return -ENODEV; 702 } 703 base += I10NM_GET_HBM_IMC_MMIO_OFFSET(reg); 704 705 lmc = res_cfg->ddr_imc_num; 706 707 for (i = 0; i < res_cfg->hbm_imc_num; i++) { 708 mdev = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->hbm_mdev_bdf.bus], 709 res_cfg->hbm_mdev_bdf.dev + i / 4, 710 res_cfg->hbm_mdev_bdf.fun + i % 4); 711 712 if (i == 0 && !mdev) { 713 i10nm_printk(KERN_ERR, "No hbm mc found\n"); 714 return -ENODEV; 715 } 716 if (!mdev) 717 continue; 718 719 d->imc[lmc].mdev = mdev; 720 off = i * I10NM_HBM_IMC_MMIO_SIZE; 721 722 edac_dbg(2, "hbm mc%d mmio base 0x%llx size 0x%x\n", 723 lmc, base + off, I10NM_HBM_IMC_MMIO_SIZE); 724 725 mbase = ioremap(base + off, I10NM_HBM_IMC_MMIO_SIZE); 726 if (!mbase) { 727 pci_dev_put(d->imc[lmc].mdev); 728 d->imc[lmc].mdev = NULL; 729 730 i10nm_printk(KERN_ERR, "Failed to ioremap for hbm mc 0x%llx\n", 731 base + off); 732 return -ENOMEM; 733 } 734 735 d->imc[lmc].mbase = mbase; 736 d->imc[lmc].hbm_mc = true; 737 738 mcmtr = I10NM_GET_MCMTR(&d->imc[lmc], 0); 739 if (!I10NM_IS_HBM_IMC(mcmtr)) { 740 iounmap(d->imc[lmc].mbase); 741 d->imc[lmc].mbase = NULL; 742 d->imc[lmc].hbm_mc = false; 743 pci_dev_put(d->imc[lmc].mdev); 744 d->imc[lmc].mdev = NULL; 745 746 i10nm_printk(KERN_ERR, "This isn't an hbm mc!\n"); 747 return -ENODEV; 748 } 749 750 lmc++; 751 } 752 } 753 754 return 0; 755 } 756 757 static struct res_config i10nm_cfg0 = { 758 .type = I10NM, 759 .decs_did = 0x3452, 760 .busno_cfg_offset = 0xcc, 761 .ddr_imc_num = 4, 762 .ddr_chan_num = 2, 763 .ddr_dimm_num = 2, 764 .ddr_chan_mmio_sz = 0x4000, 765 .sad_all_bdf = {1, 29, 0}, 766 .pcu_cr3_bdf = {1, 30, 3}, 767 .util_all_bdf = {1, 29, 1}, 768 .uracu_bdf = {0, 0, 1}, 769 .ddr_mdev_bdf = {0, 12, 0}, 770 .hbm_mdev_bdf = {0, 12, 1}, 771 .sad_all_offset = 0x108, 772 .reg_rrl_ddr[0] = &icx_reg_rrl_ddr, 773 }; 774 775 static struct res_config i10nm_cfg1 = { 776 .type = I10NM, 777 .decs_did = 0x3452, 778 .busno_cfg_offset = 0xd0, 779 .ddr_imc_num = 4, 780 .ddr_chan_num = 2, 781 .ddr_dimm_num = 2, 782 .ddr_chan_mmio_sz = 0x4000, 783 .sad_all_bdf = {1, 29, 0}, 784 .pcu_cr3_bdf = {1, 30, 3}, 785 .util_all_bdf = {1, 29, 1}, 786 .uracu_bdf = {0, 0, 1}, 787 .ddr_mdev_bdf = {0, 12, 0}, 788 .hbm_mdev_bdf = {0, 12, 1}, 789 .sad_all_offset = 0x108, 790 .reg_rrl_ddr[0] = &icx_reg_rrl_ddr, 791 }; 792 793 static struct res_config spr_cfg = { 794 .type = SPR, 795 .decs_did = 0x3252, 796 .busno_cfg_offset = 0xd0, 797 .ddr_imc_num = 4, 798 .ddr_chan_num = 2, 799 .ddr_dimm_num = 2, 800 .hbm_imc_num = 16, 801 .hbm_chan_num = 2, 802 .hbm_dimm_num = 1, 803 .ddr_chan_mmio_sz = 0x8000, 804 .hbm_chan_mmio_sz = 0x4000, 805 .support_ddr5 = true, 806 .sad_all_bdf = {1, 10, 0}, 807 .pcu_cr3_bdf = {1, 30, 3}, 808 .util_all_bdf = {1, 29, 1}, 809 .uracu_bdf = {0, 0, 1}, 810 .ddr_mdev_bdf = {0, 12, 0}, 811 .hbm_mdev_bdf = {0, 12, 1}, 812 .sad_all_offset = 0x300, 813 .reg_rrl_ddr[0] = &spr_reg_rrl_ddr, 814 .reg_rrl_hbm[0] = &spr_reg_rrl_hbm_pch0, 815 .reg_rrl_hbm[1] = &spr_reg_rrl_hbm_pch1, 816 }; 817 818 static struct res_config gnr_cfg = { 819 .type = GNR, 820 .decs_did = 0x3252, 821 .busno_cfg_offset = 0xd0, 822 .ddr_imc_num = 12, 823 .ddr_chan_num = 1, 824 .ddr_dimm_num = 2, 825 .ddr_chan_mmio_sz = 0x4000, 826 .support_ddr5 = true, 827 .sad_all_bdf = {0, 13, 0}, 828 .pcu_cr3_bdf = {0, 5, 0}, 829 .util_all_bdf = {0, 13, 1}, 830 .uracu_bdf = {0, 0, 1}, 831 .ddr_mdev_bdf = {0, 5, 1}, 832 .sad_all_offset = 0x300, 833 .reg_rrl_ddr[0] = &gnr_reg_rrl_ddr, 834 }; 835 836 static const struct x86_cpu_id i10nm_cpuids[] = { 837 X86_MATCH_VFM_STEPS(INTEL_ATOM_TREMONT_D, X86_STEP_MIN, 0x3, &i10nm_cfg0), 838 X86_MATCH_VFM_STEPS(INTEL_ATOM_TREMONT_D, 0x4, X86_STEP_MAX, &i10nm_cfg1), 839 X86_MATCH_VFM_STEPS(INTEL_ICELAKE_X, X86_STEP_MIN, 0x3, &i10nm_cfg0), 840 X86_MATCH_VFM_STEPS(INTEL_ICELAKE_X, 0x4, X86_STEP_MAX, &i10nm_cfg1), 841 X86_MATCH_VFM( INTEL_ICELAKE_D, &i10nm_cfg1), 842 843 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &spr_cfg), 844 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &spr_cfg), 845 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &gnr_cfg), 846 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &gnr_cfg), 847 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &gnr_cfg), 848 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &gnr_cfg), 849 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &gnr_cfg), 850 {} 851 }; 852 MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids); 853 854 static bool i10nm_check_ecc(struct skx_imc *imc, int chan) 855 { 856 u32 mcmtr; 857 858 mcmtr = I10NM_GET_MCMTR(imc, chan); 859 edac_dbg(1, "ch%d mcmtr reg %x\n", chan, mcmtr); 860 861 return !!GET_BITFIELD(mcmtr, 2, 2); 862 } 863 864 static bool i10nm_channel_disabled(struct skx_imc *imc, int chan) 865 { 866 u32 mcmtr = I10NM_GET_MCMTR(imc, chan); 867 868 edac_dbg(1, "mc%d ch%d mcmtr reg %x\n", imc->mc, chan, mcmtr); 869 870 return (mcmtr == ~0 || GET_BITFIELD(mcmtr, 18, 18)); 871 } 872 873 static int i10nm_get_dimm_config(struct mem_ctl_info *mci, 874 struct res_config *cfg) 875 { 876 struct skx_pvt *pvt = mci->pvt_info; 877 struct skx_imc *imc = pvt->imc; 878 u32 mtr, mcddrtcfg = 0; 879 struct dimm_info *dimm; 880 int i, j, ndimms; 881 882 for (i = 0; i < imc->num_channels; i++) { 883 if (!imc->mbase) 884 continue; 885 886 if (i10nm_channel_disabled(imc, i)) { 887 edac_dbg(1, "mc%d ch%d is disabled.\n", imc->mc, i); 888 continue; 889 } 890 891 ndimms = 0; 892 893 if (res_cfg->type != GNR) 894 mcddrtcfg = I10NM_GET_MCDDRTCFG(imc, i); 895 896 for (j = 0; j < imc->num_dimms; j++) { 897 dimm = edac_get_dimm(mci, i, j, 0); 898 mtr = I10NM_GET_DIMMMTR(imc, i, j); 899 edac_dbg(1, "dimmmtr 0x%x mcddrtcfg 0x%x (mc%d ch%d dimm%d)\n", 900 mtr, mcddrtcfg, imc->mc, i, j); 901 902 if (IS_DIMM_PRESENT(mtr)) 903 ndimms += skx_get_dimm_info(mtr, 0, 0, dimm, 904 imc, i, j, cfg); 905 else if (IS_NVDIMM_PRESENT(mcddrtcfg, j)) 906 ndimms += skx_get_nvdimm_info(dimm, imc, i, j, 907 EDAC_MOD_STR); 908 } 909 if (ndimms && !i10nm_check_ecc(imc, i)) { 910 i10nm_printk(KERN_ERR, "ECC is disabled on imc %d channel %d\n", 911 imc->mc, i); 912 return -ENODEV; 913 } 914 } 915 916 return 0; 917 } 918 919 static struct notifier_block i10nm_mce_dec = { 920 .notifier_call = skx_mce_check_error, 921 .priority = MCE_PRIO_EDAC, 922 }; 923 924 static int __init i10nm_init(void) 925 { 926 u8 mc = 0, src_id = 0; 927 const struct x86_cpu_id *id; 928 struct res_config *cfg; 929 const char *owner; 930 struct skx_dev *d; 931 int rc, i, off[3] = {0xd0, 0xc8, 0xcc}; 932 u64 tolm, tohm; 933 int imc_num; 934 935 edac_dbg(2, "\n"); 936 937 if (ghes_get_devices()) 938 return -EBUSY; 939 940 owner = edac_get_owner(); 941 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 942 return -EBUSY; 943 944 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 945 return -ENODEV; 946 947 id = x86_match_cpu(i10nm_cpuids); 948 if (!id) 949 return -ENODEV; 950 951 cfg = (struct res_config *)id->driver_data; 952 skx_set_res_cfg(cfg); 953 res_cfg = cfg; 954 955 rc = skx_get_hi_lo(0x09a2, off, &tolm, &tohm); 956 if (rc) 957 return rc; 958 959 rc = skx_get_all_bus_mappings(cfg, &i10nm_edac_list); 960 if (rc < 0) 961 goto fail; 962 if (rc == 0) { 963 i10nm_printk(KERN_ERR, "No memory controllers found\n"); 964 return -ENODEV; 965 } 966 967 rc = i10nm_get_imc_num(cfg); 968 if (rc < 0) 969 goto fail; 970 971 mem_cfg_2lm = i10nm_check_2lm(cfg); 972 skx_set_mem_cfg(mem_cfg_2lm); 973 974 rc = i10nm_get_ddr_munits(); 975 976 if (i10nm_get_hbm_munits() && rc) 977 goto fail; 978 979 imc_num = res_cfg->ddr_imc_num + res_cfg->hbm_imc_num; 980 981 list_for_each_entry(d, i10nm_edac_list, list) { 982 rc = skx_get_src_id(d, 0xf8, &src_id); 983 if (rc < 0) 984 goto fail; 985 986 edac_dbg(2, "src_id = %d\n", src_id); 987 for (i = 0; i < imc_num; i++) { 988 if (!d->imc[i].mdev) 989 continue; 990 991 d->imc[i].mc = mc++; 992 d->imc[i].lmc = i; 993 d->imc[i].src_id = src_id; 994 if (d->imc[i].hbm_mc) { 995 d->imc[i].chan_mmio_sz = cfg->hbm_chan_mmio_sz; 996 d->imc[i].num_channels = cfg->hbm_chan_num; 997 d->imc[i].num_dimms = cfg->hbm_dimm_num; 998 } else { 999 d->imc[i].chan_mmio_sz = cfg->ddr_chan_mmio_sz; 1000 d->imc[i].num_channels = cfg->ddr_chan_num; 1001 d->imc[i].num_dimms = cfg->ddr_dimm_num; 1002 } 1003 1004 rc = skx_register_mci(&d->imc[i], &d->imc[i].mdev->dev, 1005 pci_name(d->imc[i].mdev), 1006 "Intel_10nm Socket", EDAC_MOD_STR, 1007 i10nm_get_dimm_config, cfg); 1008 if (rc < 0) 1009 goto fail; 1010 } 1011 } 1012 1013 rc = skx_adxl_get(); 1014 if (rc) { 1015 /* Decoding errors via MCA banks for 2LM isn't supported yet */ 1016 if (rc != -ENODEV || mem_cfg_2lm) 1017 goto fail; 1018 i10nm_printk(KERN_INFO, "ADXL not found, falling back to MCA-based decoding.\n"); 1019 no_adxl = true; 1020 decoding_via_mca = true; 1021 } 1022 1023 opstate_init(); 1024 mce_register_decode_chain(&i10nm_mce_dec); 1025 skx_setup_debug("i10nm_test"); 1026 1027 res_cfg->rrl_ctrl_mode = retry_rd_err_log; 1028 if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) { 1029 skx_set_show_rrl(skx_show_rrl); 1030 if (retry_rd_err_log == RRL_CTRL_LINUX) 1031 skx_enable_rrl(true); 1032 } 1033 1034 skx_set_decode(i10nm_mc_decode); 1035 1036 i10nm_printk(KERN_INFO, "%s\n", I10NM_REVISION); 1037 1038 return 0; 1039 fail: 1040 skx_remove(); 1041 return rc; 1042 } 1043 1044 static void __exit i10nm_exit(void) 1045 { 1046 edac_dbg(2, "\n"); 1047 1048 skx_set_decode(NULL); 1049 1050 if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) { 1051 if (retry_rd_err_log == RRL_CTRL_LINUX) 1052 skx_enable_rrl(false); 1053 skx_set_show_rrl(NULL); 1054 } 1055 1056 skx_teardown_debug(); 1057 mce_unregister_decode_chain(&i10nm_mce_dec); 1058 if (!no_adxl) 1059 skx_adxl_put(); 1060 skx_remove(); 1061 } 1062 1063 module_init(i10nm_init); 1064 module_exit(i10nm_exit); 1065 1066 static int set_decoding_via_mca(const char *buf, const struct kernel_param *kp) 1067 { 1068 unsigned long val; 1069 int ret; 1070 1071 ret = kstrtoul(buf, 0, &val); 1072 1073 if (ret || val > 1) 1074 return -EINVAL; 1075 1076 if (val && mem_cfg_2lm) { 1077 i10nm_printk(KERN_NOTICE, "Decoding errors via MCA banks for 2LM isn't supported yet\n"); 1078 return -EIO; 1079 } 1080 1081 ret = param_set_int(buf, kp); 1082 1083 return ret; 1084 } 1085 1086 static const struct kernel_param_ops decoding_via_mca_param_ops = { 1087 .set = set_decoding_via_mca, 1088 .get = param_get_int, 1089 }; 1090 1091 module_param_cb(decoding_via_mca, &decoding_via_mca_param_ops, &decoding_via_mca, 0644); 1092 MODULE_PARM_DESC(decoding_via_mca, "decoding_via_mca: 0=off(default), 1=enable"); 1093 1094 module_param(retry_rd_err_log, int, 0444); 1095 MODULE_PARM_DESC(retry_rd_err_log, "retry_rd_err_log: 0=off(default), 1=bios(Linux doesn't reset any control bits, but just reports values.), 2=linux(Linux tries to take control and resets mode bits, clear valid/UC bits after reading.)"); 1096 1097 MODULE_LICENSE("GPL v2"); 1098 MODULE_DESCRIPTION("MC Driver for Intel 10nm server processors"); 1099