1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/list_sort.h> 14 #include <linux/libnvdimm.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/ndctl.h> 18 #include <linux/sysfs.h> 19 #include <linux/delay.h> 20 #include <linux/list.h> 21 #include <linux/acpi.h> 22 #include <linux/sort.h> 23 #include <linux/pmem.h> 24 #include <linux/io.h> 25 #include <linux/nd.h> 26 #include <asm/cacheflush.h> 27 #include "nfit.h" 28 29 /* 30 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 31 * irrelevant. 32 */ 33 #include <linux/io-64-nonatomic-hi-lo.h> 34 35 static bool force_enable_dimms; 36 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); 37 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); 38 39 static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT; 40 module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR); 41 MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds"); 42 43 /* after three payloads of overflow, it's dead jim */ 44 static unsigned int scrub_overflow_abort = 3; 45 module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR); 46 MODULE_PARM_DESC(scrub_overflow_abort, 47 "Number of times we overflow ARS results before abort"); 48 49 static bool disable_vendor_specific; 50 module_param(disable_vendor_specific, bool, S_IRUGO); 51 MODULE_PARM_DESC(disable_vendor_specific, 52 "Limit commands to the publicly specified set\n"); 53 54 LIST_HEAD(acpi_descs); 55 DEFINE_MUTEX(acpi_desc_lock); 56 57 static struct workqueue_struct *nfit_wq; 58 59 struct nfit_table_prev { 60 struct list_head spas; 61 struct list_head memdevs; 62 struct list_head dcrs; 63 struct list_head bdws; 64 struct list_head idts; 65 struct list_head flushes; 66 }; 67 68 static u8 nfit_uuid[NFIT_UUID_MAX][16]; 69 70 const u8 *to_nfit_uuid(enum nfit_uuids id) 71 { 72 return nfit_uuid[id]; 73 } 74 EXPORT_SYMBOL(to_nfit_uuid); 75 76 static struct acpi_nfit_desc *to_acpi_nfit_desc( 77 struct nvdimm_bus_descriptor *nd_desc) 78 { 79 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); 80 } 81 82 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) 83 { 84 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 85 86 /* 87 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct 88 * acpi_device. 89 */ 90 if (!nd_desc->provider_name 91 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) 92 return NULL; 93 94 return to_acpi_device(acpi_desc->dev); 95 } 96 97 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status) 98 { 99 struct nd_cmd_clear_error *clear_err; 100 struct nd_cmd_ars_status *ars_status; 101 u16 flags; 102 103 switch (cmd) { 104 case ND_CMD_ARS_CAP: 105 if ((status & 0xffff) == NFIT_ARS_CAP_NONE) 106 return -ENOTTY; 107 108 /* Command failed */ 109 if (status & 0xffff) 110 return -EIO; 111 112 /* No supported scan types for this range */ 113 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; 114 if ((status >> 16 & flags) == 0) 115 return -ENOTTY; 116 return 0; 117 case ND_CMD_ARS_START: 118 /* ARS is in progress */ 119 if ((status & 0xffff) == NFIT_ARS_START_BUSY) 120 return -EBUSY; 121 122 /* Command failed */ 123 if (status & 0xffff) 124 return -EIO; 125 return 0; 126 case ND_CMD_ARS_STATUS: 127 ars_status = buf; 128 /* Command failed */ 129 if (status & 0xffff) 130 return -EIO; 131 /* Check extended status (Upper two bytes) */ 132 if (status == NFIT_ARS_STATUS_DONE) 133 return 0; 134 135 /* ARS is in progress */ 136 if (status == NFIT_ARS_STATUS_BUSY) 137 return -EBUSY; 138 139 /* No ARS performed for the current boot */ 140 if (status == NFIT_ARS_STATUS_NONE) 141 return -EAGAIN; 142 143 /* 144 * ARS interrupted, either we overflowed or some other 145 * agent wants the scan to stop. If we didn't overflow 146 * then just continue with the returned results. 147 */ 148 if (status == NFIT_ARS_STATUS_INTR) { 149 if (ars_status->out_length >= 40 && (ars_status->flags 150 & NFIT_ARS_F_OVERFLOW)) 151 return -ENOSPC; 152 return 0; 153 } 154 155 /* Unknown status */ 156 if (status >> 16) 157 return -EIO; 158 return 0; 159 case ND_CMD_CLEAR_ERROR: 160 clear_err = buf; 161 if (status & 0xffff) 162 return -EIO; 163 if (!clear_err->cleared) 164 return -EIO; 165 if (clear_err->length > clear_err->cleared) 166 return clear_err->cleared; 167 return 0; 168 default: 169 break; 170 } 171 172 /* all other non-zero status results in an error */ 173 if (status) 174 return -EIO; 175 return 0; 176 } 177 178 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 179 u32 status) 180 { 181 if (!nvdimm) 182 return xlat_bus_status(buf, cmd, status); 183 if (status) 184 return -EIO; 185 return 0; 186 } 187 188 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, 189 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) 190 { 191 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 192 union acpi_object in_obj, in_buf, *out_obj; 193 const struct nd_cmd_desc *desc = NULL; 194 struct device *dev = acpi_desc->dev; 195 struct nd_cmd_pkg *call_pkg = NULL; 196 const char *cmd_name, *dimm_name; 197 unsigned long cmd_mask, dsm_mask; 198 u32 offset, fw_status = 0; 199 acpi_handle handle; 200 unsigned int func; 201 const u8 *uuid; 202 int rc, i; 203 204 func = cmd; 205 if (cmd == ND_CMD_CALL) { 206 call_pkg = buf; 207 func = call_pkg->nd_command; 208 } 209 210 if (nvdimm) { 211 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 212 struct acpi_device *adev = nfit_mem->adev; 213 214 if (!adev) 215 return -ENOTTY; 216 if (call_pkg && nfit_mem->family != call_pkg->nd_family) 217 return -ENOTTY; 218 219 dimm_name = nvdimm_name(nvdimm); 220 cmd_name = nvdimm_cmd_name(cmd); 221 cmd_mask = nvdimm_cmd_mask(nvdimm); 222 dsm_mask = nfit_mem->dsm_mask; 223 desc = nd_cmd_dimm_desc(cmd); 224 uuid = to_nfit_uuid(nfit_mem->family); 225 handle = adev->handle; 226 } else { 227 struct acpi_device *adev = to_acpi_dev(acpi_desc); 228 229 cmd_name = nvdimm_bus_cmd_name(cmd); 230 cmd_mask = nd_desc->cmd_mask; 231 dsm_mask = cmd_mask; 232 desc = nd_cmd_bus_desc(cmd); 233 uuid = to_nfit_uuid(NFIT_DEV_BUS); 234 handle = adev->handle; 235 dimm_name = "bus"; 236 } 237 238 if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) 239 return -ENOTTY; 240 241 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) 242 return -ENOTTY; 243 244 in_obj.type = ACPI_TYPE_PACKAGE; 245 in_obj.package.count = 1; 246 in_obj.package.elements = &in_buf; 247 in_buf.type = ACPI_TYPE_BUFFER; 248 in_buf.buffer.pointer = buf; 249 in_buf.buffer.length = 0; 250 251 /* libnvdimm has already validated the input envelope */ 252 for (i = 0; i < desc->in_num; i++) 253 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, 254 i, buf); 255 256 if (call_pkg) { 257 /* skip over package wrapper */ 258 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; 259 in_buf.buffer.length = call_pkg->nd_size_in; 260 } 261 262 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) { 263 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n", 264 __func__, dimm_name, cmd, func, 265 in_buf.buffer.length); 266 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, 267 in_buf.buffer.pointer, 268 min_t(u32, 256, in_buf.buffer.length), true); 269 } 270 271 out_obj = acpi_evaluate_dsm(handle, uuid, 1, func, &in_obj); 272 if (!out_obj) { 273 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name, 274 cmd_name); 275 return -EINVAL; 276 } 277 278 if (call_pkg) { 279 call_pkg->nd_fw_size = out_obj->buffer.length; 280 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, 281 out_obj->buffer.pointer, 282 min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); 283 284 ACPI_FREE(out_obj); 285 /* 286 * Need to support FW function w/o known size in advance. 287 * Caller can determine required size based upon nd_fw_size. 288 * If we return an error (like elsewhere) then caller wouldn't 289 * be able to rely upon data returned to make calculation. 290 */ 291 return 0; 292 } 293 294 if (out_obj->package.type != ACPI_TYPE_BUFFER) { 295 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n", 296 __func__, dimm_name, cmd_name, out_obj->type); 297 rc = -EINVAL; 298 goto out; 299 } 300 301 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) { 302 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, 303 dimm_name, cmd_name, out_obj->buffer.length); 304 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 305 4, out_obj->buffer.pointer, min_t(u32, 128, 306 out_obj->buffer.length), true); 307 } 308 309 for (i = 0, offset = 0; i < desc->out_num; i++) { 310 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, 311 (u32 *) out_obj->buffer.pointer, 312 out_obj->buffer.length - offset); 313 314 if (offset + out_size > out_obj->buffer.length) { 315 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n", 316 __func__, dimm_name, cmd_name, i); 317 break; 318 } 319 320 if (in_buf.buffer.length + offset + out_size > buf_len) { 321 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n", 322 __func__, dimm_name, cmd_name, i); 323 rc = -ENXIO; 324 goto out; 325 } 326 memcpy(buf + in_buf.buffer.length + offset, 327 out_obj->buffer.pointer + offset, out_size); 328 offset += out_size; 329 } 330 331 /* 332 * Set fw_status for all the commands with a known format to be 333 * later interpreted by xlat_status(). 334 */ 335 if (i >= 1 && ((cmd >= ND_CMD_ARS_CAP && cmd <= ND_CMD_CLEAR_ERROR) 336 || (cmd >= ND_CMD_SMART && cmd <= ND_CMD_VENDOR))) 337 fw_status = *(u32 *) out_obj->buffer.pointer; 338 339 if (offset + in_buf.buffer.length < buf_len) { 340 if (i >= 1) { 341 /* 342 * status valid, return the number of bytes left 343 * unfilled in the output buffer 344 */ 345 rc = buf_len - offset - in_buf.buffer.length; 346 if (cmd_rc) 347 *cmd_rc = xlat_status(nvdimm, buf, cmd, 348 fw_status); 349 } else { 350 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", 351 __func__, dimm_name, cmd_name, buf_len, 352 offset); 353 rc = -ENXIO; 354 } 355 } else { 356 rc = 0; 357 if (cmd_rc) 358 *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status); 359 } 360 361 out: 362 ACPI_FREE(out_obj); 363 364 return rc; 365 } 366 EXPORT_SYMBOL_GPL(acpi_nfit_ctl); 367 368 static const char *spa_type_name(u16 type) 369 { 370 static const char *to_name[] = { 371 [NFIT_SPA_VOLATILE] = "volatile", 372 [NFIT_SPA_PM] = "pmem", 373 [NFIT_SPA_DCR] = "dimm-control-region", 374 [NFIT_SPA_BDW] = "block-data-window", 375 [NFIT_SPA_VDISK] = "volatile-disk", 376 [NFIT_SPA_VCD] = "volatile-cd", 377 [NFIT_SPA_PDISK] = "persistent-disk", 378 [NFIT_SPA_PCD] = "persistent-cd", 379 380 }; 381 382 if (type > NFIT_SPA_PCD) 383 return "unknown"; 384 385 return to_name[type]; 386 } 387 388 int nfit_spa_type(struct acpi_nfit_system_address *spa) 389 { 390 int i; 391 392 for (i = 0; i < NFIT_UUID_MAX; i++) 393 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0) 394 return i; 395 return -1; 396 } 397 398 static bool add_spa(struct acpi_nfit_desc *acpi_desc, 399 struct nfit_table_prev *prev, 400 struct acpi_nfit_system_address *spa) 401 { 402 struct device *dev = acpi_desc->dev; 403 struct nfit_spa *nfit_spa; 404 405 if (spa->header.length != sizeof(*spa)) 406 return false; 407 408 list_for_each_entry(nfit_spa, &prev->spas, list) { 409 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { 410 list_move_tail(&nfit_spa->list, &acpi_desc->spas); 411 return true; 412 } 413 } 414 415 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), 416 GFP_KERNEL); 417 if (!nfit_spa) 418 return false; 419 INIT_LIST_HEAD(&nfit_spa->list); 420 memcpy(nfit_spa->spa, spa, sizeof(*spa)); 421 list_add_tail(&nfit_spa->list, &acpi_desc->spas); 422 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__, 423 spa->range_index, 424 spa_type_name(nfit_spa_type(spa))); 425 return true; 426 } 427 428 static bool add_memdev(struct acpi_nfit_desc *acpi_desc, 429 struct nfit_table_prev *prev, 430 struct acpi_nfit_memory_map *memdev) 431 { 432 struct device *dev = acpi_desc->dev; 433 struct nfit_memdev *nfit_memdev; 434 435 if (memdev->header.length != sizeof(*memdev)) 436 return false; 437 438 list_for_each_entry(nfit_memdev, &prev->memdevs, list) 439 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { 440 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); 441 return true; 442 } 443 444 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), 445 GFP_KERNEL); 446 if (!nfit_memdev) 447 return false; 448 INIT_LIST_HEAD(&nfit_memdev->list); 449 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); 450 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); 451 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n", 452 __func__, memdev->device_handle, memdev->range_index, 453 memdev->region_index); 454 return true; 455 } 456 457 /* 458 * An implementation may provide a truncated control region if no block windows 459 * are defined. 460 */ 461 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) 462 { 463 if (dcr->header.length < offsetof(struct acpi_nfit_control_region, 464 window_size)) 465 return 0; 466 if (dcr->windows) 467 return sizeof(*dcr); 468 return offsetof(struct acpi_nfit_control_region, window_size); 469 } 470 471 static bool add_dcr(struct acpi_nfit_desc *acpi_desc, 472 struct nfit_table_prev *prev, 473 struct acpi_nfit_control_region *dcr) 474 { 475 struct device *dev = acpi_desc->dev; 476 struct nfit_dcr *nfit_dcr; 477 478 if (!sizeof_dcr(dcr)) 479 return false; 480 481 list_for_each_entry(nfit_dcr, &prev->dcrs, list) 482 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { 483 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); 484 return true; 485 } 486 487 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), 488 GFP_KERNEL); 489 if (!nfit_dcr) 490 return false; 491 INIT_LIST_HEAD(&nfit_dcr->list); 492 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); 493 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); 494 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__, 495 dcr->region_index, dcr->windows); 496 return true; 497 } 498 499 static bool add_bdw(struct acpi_nfit_desc *acpi_desc, 500 struct nfit_table_prev *prev, 501 struct acpi_nfit_data_region *bdw) 502 { 503 struct device *dev = acpi_desc->dev; 504 struct nfit_bdw *nfit_bdw; 505 506 if (bdw->header.length != sizeof(*bdw)) 507 return false; 508 list_for_each_entry(nfit_bdw, &prev->bdws, list) 509 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { 510 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); 511 return true; 512 } 513 514 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), 515 GFP_KERNEL); 516 if (!nfit_bdw) 517 return false; 518 INIT_LIST_HEAD(&nfit_bdw->list); 519 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); 520 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); 521 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__, 522 bdw->region_index, bdw->windows); 523 return true; 524 } 525 526 static size_t sizeof_idt(struct acpi_nfit_interleave *idt) 527 { 528 if (idt->header.length < sizeof(*idt)) 529 return 0; 530 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); 531 } 532 533 static bool add_idt(struct acpi_nfit_desc *acpi_desc, 534 struct nfit_table_prev *prev, 535 struct acpi_nfit_interleave *idt) 536 { 537 struct device *dev = acpi_desc->dev; 538 struct nfit_idt *nfit_idt; 539 540 if (!sizeof_idt(idt)) 541 return false; 542 543 list_for_each_entry(nfit_idt, &prev->idts, list) { 544 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) 545 continue; 546 547 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { 548 list_move_tail(&nfit_idt->list, &acpi_desc->idts); 549 return true; 550 } 551 } 552 553 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), 554 GFP_KERNEL); 555 if (!nfit_idt) 556 return false; 557 INIT_LIST_HEAD(&nfit_idt->list); 558 memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); 559 list_add_tail(&nfit_idt->list, &acpi_desc->idts); 560 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__, 561 idt->interleave_index, idt->line_count); 562 return true; 563 } 564 565 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) 566 { 567 if (flush->header.length < sizeof(*flush)) 568 return 0; 569 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); 570 } 571 572 static bool add_flush(struct acpi_nfit_desc *acpi_desc, 573 struct nfit_table_prev *prev, 574 struct acpi_nfit_flush_address *flush) 575 { 576 struct device *dev = acpi_desc->dev; 577 struct nfit_flush *nfit_flush; 578 579 if (!sizeof_flush(flush)) 580 return false; 581 582 list_for_each_entry(nfit_flush, &prev->flushes, list) { 583 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) 584 continue; 585 586 if (memcmp(nfit_flush->flush, flush, 587 sizeof_flush(flush)) == 0) { 588 list_move_tail(&nfit_flush->list, &acpi_desc->flushes); 589 return true; 590 } 591 } 592 593 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) 594 + sizeof_flush(flush), GFP_KERNEL); 595 if (!nfit_flush) 596 return false; 597 INIT_LIST_HEAD(&nfit_flush->list); 598 memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); 599 list_add_tail(&nfit_flush->list, &acpi_desc->flushes); 600 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__, 601 flush->device_handle, flush->hint_count); 602 return true; 603 } 604 605 static void *add_table(struct acpi_nfit_desc *acpi_desc, 606 struct nfit_table_prev *prev, void *table, const void *end) 607 { 608 struct device *dev = acpi_desc->dev; 609 struct acpi_nfit_header *hdr; 610 void *err = ERR_PTR(-ENOMEM); 611 612 if (table >= end) 613 return NULL; 614 615 hdr = table; 616 if (!hdr->length) { 617 dev_warn(dev, "found a zero length table '%d' parsing nfit\n", 618 hdr->type); 619 return NULL; 620 } 621 622 switch (hdr->type) { 623 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: 624 if (!add_spa(acpi_desc, prev, table)) 625 return err; 626 break; 627 case ACPI_NFIT_TYPE_MEMORY_MAP: 628 if (!add_memdev(acpi_desc, prev, table)) 629 return err; 630 break; 631 case ACPI_NFIT_TYPE_CONTROL_REGION: 632 if (!add_dcr(acpi_desc, prev, table)) 633 return err; 634 break; 635 case ACPI_NFIT_TYPE_DATA_REGION: 636 if (!add_bdw(acpi_desc, prev, table)) 637 return err; 638 break; 639 case ACPI_NFIT_TYPE_INTERLEAVE: 640 if (!add_idt(acpi_desc, prev, table)) 641 return err; 642 break; 643 case ACPI_NFIT_TYPE_FLUSH_ADDRESS: 644 if (!add_flush(acpi_desc, prev, table)) 645 return err; 646 break; 647 case ACPI_NFIT_TYPE_SMBIOS: 648 dev_dbg(dev, "%s: smbios\n", __func__); 649 break; 650 default: 651 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); 652 break; 653 } 654 655 return table + hdr->length; 656 } 657 658 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, 659 struct nfit_mem *nfit_mem) 660 { 661 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 662 u16 dcr = nfit_mem->dcr->region_index; 663 struct nfit_spa *nfit_spa; 664 665 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 666 u16 range_index = nfit_spa->spa->range_index; 667 int type = nfit_spa_type(nfit_spa->spa); 668 struct nfit_memdev *nfit_memdev; 669 670 if (type != NFIT_SPA_BDW) 671 continue; 672 673 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 674 if (nfit_memdev->memdev->range_index != range_index) 675 continue; 676 if (nfit_memdev->memdev->device_handle != device_handle) 677 continue; 678 if (nfit_memdev->memdev->region_index != dcr) 679 continue; 680 681 nfit_mem->spa_bdw = nfit_spa->spa; 682 return; 683 } 684 } 685 686 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", 687 nfit_mem->spa_dcr->range_index); 688 nfit_mem->bdw = NULL; 689 } 690 691 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, 692 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) 693 { 694 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; 695 struct nfit_memdev *nfit_memdev; 696 struct nfit_bdw *nfit_bdw; 697 struct nfit_idt *nfit_idt; 698 u16 idt_idx, range_index; 699 700 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { 701 if (nfit_bdw->bdw->region_index != dcr) 702 continue; 703 nfit_mem->bdw = nfit_bdw->bdw; 704 break; 705 } 706 707 if (!nfit_mem->bdw) 708 return; 709 710 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); 711 712 if (!nfit_mem->spa_bdw) 713 return; 714 715 range_index = nfit_mem->spa_bdw->range_index; 716 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 717 if (nfit_memdev->memdev->range_index != range_index || 718 nfit_memdev->memdev->region_index != dcr) 719 continue; 720 nfit_mem->memdev_bdw = nfit_memdev->memdev; 721 idt_idx = nfit_memdev->memdev->interleave_index; 722 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 723 if (nfit_idt->idt->interleave_index != idt_idx) 724 continue; 725 nfit_mem->idt_bdw = nfit_idt->idt; 726 break; 727 } 728 break; 729 } 730 } 731 732 static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc, 733 struct acpi_nfit_system_address *spa) 734 { 735 struct nfit_mem *nfit_mem, *found; 736 struct nfit_memdev *nfit_memdev; 737 int type = nfit_spa_type(spa); 738 739 switch (type) { 740 case NFIT_SPA_DCR: 741 case NFIT_SPA_PM: 742 break; 743 default: 744 return 0; 745 } 746 747 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 748 struct nfit_flush *nfit_flush; 749 struct nfit_dcr *nfit_dcr; 750 u32 device_handle; 751 u16 dcr; 752 753 if (nfit_memdev->memdev->range_index != spa->range_index) 754 continue; 755 found = NULL; 756 dcr = nfit_memdev->memdev->region_index; 757 device_handle = nfit_memdev->memdev->device_handle; 758 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 759 if (__to_nfit_memdev(nfit_mem)->device_handle 760 == device_handle) { 761 found = nfit_mem; 762 break; 763 } 764 765 if (found) 766 nfit_mem = found; 767 else { 768 nfit_mem = devm_kzalloc(acpi_desc->dev, 769 sizeof(*nfit_mem), GFP_KERNEL); 770 if (!nfit_mem) 771 return -ENOMEM; 772 INIT_LIST_HEAD(&nfit_mem->list); 773 nfit_mem->acpi_desc = acpi_desc; 774 list_add(&nfit_mem->list, &acpi_desc->dimms); 775 } 776 777 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 778 if (nfit_dcr->dcr->region_index != dcr) 779 continue; 780 /* 781 * Record the control region for the dimm. For 782 * the ACPI 6.1 case, where there are separate 783 * control regions for the pmem vs blk 784 * interfaces, be sure to record the extended 785 * blk details. 786 */ 787 if (!nfit_mem->dcr) 788 nfit_mem->dcr = nfit_dcr->dcr; 789 else if (nfit_mem->dcr->windows == 0 790 && nfit_dcr->dcr->windows) 791 nfit_mem->dcr = nfit_dcr->dcr; 792 break; 793 } 794 795 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { 796 struct acpi_nfit_flush_address *flush; 797 u16 i; 798 799 if (nfit_flush->flush->device_handle != device_handle) 800 continue; 801 nfit_mem->nfit_flush = nfit_flush; 802 flush = nfit_flush->flush; 803 nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, 804 flush->hint_count 805 * sizeof(struct resource), GFP_KERNEL); 806 if (!nfit_mem->flush_wpq) 807 return -ENOMEM; 808 for (i = 0; i < flush->hint_count; i++) { 809 struct resource *res = &nfit_mem->flush_wpq[i]; 810 811 res->start = flush->hint_address[i]; 812 res->end = res->start + 8 - 1; 813 } 814 break; 815 } 816 817 if (dcr && !nfit_mem->dcr) { 818 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", 819 spa->range_index, dcr); 820 return -ENODEV; 821 } 822 823 if (type == NFIT_SPA_DCR) { 824 struct nfit_idt *nfit_idt; 825 u16 idt_idx; 826 827 /* multiple dimms may share a SPA when interleaved */ 828 nfit_mem->spa_dcr = spa; 829 nfit_mem->memdev_dcr = nfit_memdev->memdev; 830 idt_idx = nfit_memdev->memdev->interleave_index; 831 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 832 if (nfit_idt->idt->interleave_index != idt_idx) 833 continue; 834 nfit_mem->idt_dcr = nfit_idt->idt; 835 break; 836 } 837 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); 838 } else { 839 /* 840 * A single dimm may belong to multiple SPA-PM 841 * ranges, record at least one in addition to 842 * any SPA-DCR range. 843 */ 844 nfit_mem->memdev_pmem = nfit_memdev->memdev; 845 } 846 } 847 848 return 0; 849 } 850 851 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) 852 { 853 struct nfit_mem *a = container_of(_a, typeof(*a), list); 854 struct nfit_mem *b = container_of(_b, typeof(*b), list); 855 u32 handleA, handleB; 856 857 handleA = __to_nfit_memdev(a)->device_handle; 858 handleB = __to_nfit_memdev(b)->device_handle; 859 if (handleA < handleB) 860 return -1; 861 else if (handleA > handleB) 862 return 1; 863 return 0; 864 } 865 866 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) 867 { 868 struct nfit_spa *nfit_spa; 869 870 /* 871 * For each SPA-DCR or SPA-PMEM address range find its 872 * corresponding MEMDEV(s). From each MEMDEV find the 873 * corresponding DCR. Then, if we're operating on a SPA-DCR, 874 * try to find a SPA-BDW and a corresponding BDW that references 875 * the DCR. Throw it all into an nfit_mem object. Note, that 876 * BDWs are optional. 877 */ 878 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 879 int rc; 880 881 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa); 882 if (rc) 883 return rc; 884 } 885 886 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); 887 888 return 0; 889 } 890 891 static ssize_t revision_show(struct device *dev, 892 struct device_attribute *attr, char *buf) 893 { 894 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 895 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 896 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 897 898 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); 899 } 900 static DEVICE_ATTR_RO(revision); 901 902 static ssize_t hw_error_scrub_show(struct device *dev, 903 struct device_attribute *attr, char *buf) 904 { 905 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 906 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 907 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 908 909 return sprintf(buf, "%d\n", acpi_desc->scrub_mode); 910 } 911 912 /* 913 * The 'hw_error_scrub' attribute can have the following values written to it: 914 * '0': Switch to the default mode where an exception will only insert 915 * the address of the memory error into the poison and badblocks lists. 916 * '1': Enable a full scrub to happen if an exception for a memory error is 917 * received. 918 */ 919 static ssize_t hw_error_scrub_store(struct device *dev, 920 struct device_attribute *attr, const char *buf, size_t size) 921 { 922 struct nvdimm_bus_descriptor *nd_desc; 923 ssize_t rc; 924 long val; 925 926 rc = kstrtol(buf, 0, &val); 927 if (rc) 928 return rc; 929 930 device_lock(dev); 931 nd_desc = dev_get_drvdata(dev); 932 if (nd_desc) { 933 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 934 935 switch (val) { 936 case HW_ERROR_SCRUB_ON: 937 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON; 938 break; 939 case HW_ERROR_SCRUB_OFF: 940 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF; 941 break; 942 default: 943 rc = -EINVAL; 944 break; 945 } 946 } 947 device_unlock(dev); 948 if (rc) 949 return rc; 950 return size; 951 } 952 static DEVICE_ATTR_RW(hw_error_scrub); 953 954 /* 955 * This shows the number of full Address Range Scrubs that have been 956 * completed since driver load time. Userspace can wait on this using 957 * select/poll etc. A '+' at the end indicates an ARS is in progress 958 */ 959 static ssize_t scrub_show(struct device *dev, 960 struct device_attribute *attr, char *buf) 961 { 962 struct nvdimm_bus_descriptor *nd_desc; 963 ssize_t rc = -ENXIO; 964 965 device_lock(dev); 966 nd_desc = dev_get_drvdata(dev); 967 if (nd_desc) { 968 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 969 970 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, 971 (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); 972 } 973 device_unlock(dev); 974 return rc; 975 } 976 977 static ssize_t scrub_store(struct device *dev, 978 struct device_attribute *attr, const char *buf, size_t size) 979 { 980 struct nvdimm_bus_descriptor *nd_desc; 981 ssize_t rc; 982 long val; 983 984 rc = kstrtol(buf, 0, &val); 985 if (rc) 986 return rc; 987 if (val != 1) 988 return -EINVAL; 989 990 device_lock(dev); 991 nd_desc = dev_get_drvdata(dev); 992 if (nd_desc) { 993 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 994 995 rc = acpi_nfit_ars_rescan(acpi_desc); 996 } 997 device_unlock(dev); 998 if (rc) 999 return rc; 1000 return size; 1001 } 1002 static DEVICE_ATTR_RW(scrub); 1003 1004 static bool ars_supported(struct nvdimm_bus *nvdimm_bus) 1005 { 1006 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1007 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START 1008 | 1 << ND_CMD_ARS_STATUS; 1009 1010 return (nd_desc->cmd_mask & mask) == mask; 1011 } 1012 1013 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) 1014 { 1015 struct device *dev = container_of(kobj, struct device, kobj); 1016 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1017 1018 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) 1019 return 0; 1020 return a->mode; 1021 } 1022 1023 static struct attribute *acpi_nfit_attributes[] = { 1024 &dev_attr_revision.attr, 1025 &dev_attr_scrub.attr, 1026 &dev_attr_hw_error_scrub.attr, 1027 NULL, 1028 }; 1029 1030 static struct attribute_group acpi_nfit_attribute_group = { 1031 .name = "nfit", 1032 .attrs = acpi_nfit_attributes, 1033 .is_visible = nfit_visible, 1034 }; 1035 1036 static const struct attribute_group *acpi_nfit_attribute_groups[] = { 1037 &nvdimm_bus_attribute_group, 1038 &acpi_nfit_attribute_group, 1039 NULL, 1040 }; 1041 1042 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) 1043 { 1044 struct nvdimm *nvdimm = to_nvdimm(dev); 1045 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1046 1047 return __to_nfit_memdev(nfit_mem); 1048 } 1049 1050 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) 1051 { 1052 struct nvdimm *nvdimm = to_nvdimm(dev); 1053 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1054 1055 return nfit_mem->dcr; 1056 } 1057 1058 static ssize_t handle_show(struct device *dev, 1059 struct device_attribute *attr, char *buf) 1060 { 1061 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1062 1063 return sprintf(buf, "%#x\n", memdev->device_handle); 1064 } 1065 static DEVICE_ATTR_RO(handle); 1066 1067 static ssize_t phys_id_show(struct device *dev, 1068 struct device_attribute *attr, char *buf) 1069 { 1070 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1071 1072 return sprintf(buf, "%#x\n", memdev->physical_id); 1073 } 1074 static DEVICE_ATTR_RO(phys_id); 1075 1076 static ssize_t vendor_show(struct device *dev, 1077 struct device_attribute *attr, char *buf) 1078 { 1079 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1080 1081 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); 1082 } 1083 static DEVICE_ATTR_RO(vendor); 1084 1085 static ssize_t rev_id_show(struct device *dev, 1086 struct device_attribute *attr, char *buf) 1087 { 1088 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1089 1090 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); 1091 } 1092 static DEVICE_ATTR_RO(rev_id); 1093 1094 static ssize_t device_show(struct device *dev, 1095 struct device_attribute *attr, char *buf) 1096 { 1097 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1098 1099 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); 1100 } 1101 static DEVICE_ATTR_RO(device); 1102 1103 static ssize_t subsystem_vendor_show(struct device *dev, 1104 struct device_attribute *attr, char *buf) 1105 { 1106 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1107 1108 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); 1109 } 1110 static DEVICE_ATTR_RO(subsystem_vendor); 1111 1112 static ssize_t subsystem_rev_id_show(struct device *dev, 1113 struct device_attribute *attr, char *buf) 1114 { 1115 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1116 1117 return sprintf(buf, "0x%04x\n", 1118 be16_to_cpu(dcr->subsystem_revision_id)); 1119 } 1120 static DEVICE_ATTR_RO(subsystem_rev_id); 1121 1122 static ssize_t subsystem_device_show(struct device *dev, 1123 struct device_attribute *attr, char *buf) 1124 { 1125 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1126 1127 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); 1128 } 1129 static DEVICE_ATTR_RO(subsystem_device); 1130 1131 static int num_nvdimm_formats(struct nvdimm *nvdimm) 1132 { 1133 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1134 int formats = 0; 1135 1136 if (nfit_mem->memdev_pmem) 1137 formats++; 1138 if (nfit_mem->memdev_bdw) 1139 formats++; 1140 return formats; 1141 } 1142 1143 static ssize_t format_show(struct device *dev, 1144 struct device_attribute *attr, char *buf) 1145 { 1146 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1147 1148 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); 1149 } 1150 static DEVICE_ATTR_RO(format); 1151 1152 static ssize_t format1_show(struct device *dev, 1153 struct device_attribute *attr, char *buf) 1154 { 1155 u32 handle; 1156 ssize_t rc = -ENXIO; 1157 struct nfit_mem *nfit_mem; 1158 struct nfit_memdev *nfit_memdev; 1159 struct acpi_nfit_desc *acpi_desc; 1160 struct nvdimm *nvdimm = to_nvdimm(dev); 1161 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1162 1163 nfit_mem = nvdimm_provider_data(nvdimm); 1164 acpi_desc = nfit_mem->acpi_desc; 1165 handle = to_nfit_memdev(dev)->device_handle; 1166 1167 /* assumes DIMMs have at most 2 published interface codes */ 1168 mutex_lock(&acpi_desc->init_mutex); 1169 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1170 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 1171 struct nfit_dcr *nfit_dcr; 1172 1173 if (memdev->device_handle != handle) 1174 continue; 1175 1176 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1177 if (nfit_dcr->dcr->region_index != memdev->region_index) 1178 continue; 1179 if (nfit_dcr->dcr->code == dcr->code) 1180 continue; 1181 rc = sprintf(buf, "0x%04x\n", 1182 le16_to_cpu(nfit_dcr->dcr->code)); 1183 break; 1184 } 1185 if (rc != ENXIO) 1186 break; 1187 } 1188 mutex_unlock(&acpi_desc->init_mutex); 1189 return rc; 1190 } 1191 static DEVICE_ATTR_RO(format1); 1192 1193 static ssize_t formats_show(struct device *dev, 1194 struct device_attribute *attr, char *buf) 1195 { 1196 struct nvdimm *nvdimm = to_nvdimm(dev); 1197 1198 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); 1199 } 1200 static DEVICE_ATTR_RO(formats); 1201 1202 static ssize_t serial_show(struct device *dev, 1203 struct device_attribute *attr, char *buf) 1204 { 1205 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1206 1207 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); 1208 } 1209 static DEVICE_ATTR_RO(serial); 1210 1211 static ssize_t family_show(struct device *dev, 1212 struct device_attribute *attr, char *buf) 1213 { 1214 struct nvdimm *nvdimm = to_nvdimm(dev); 1215 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1216 1217 if (nfit_mem->family < 0) 1218 return -ENXIO; 1219 return sprintf(buf, "%d\n", nfit_mem->family); 1220 } 1221 static DEVICE_ATTR_RO(family); 1222 1223 static ssize_t dsm_mask_show(struct device *dev, 1224 struct device_attribute *attr, char *buf) 1225 { 1226 struct nvdimm *nvdimm = to_nvdimm(dev); 1227 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1228 1229 if (nfit_mem->family < 0) 1230 return -ENXIO; 1231 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); 1232 } 1233 static DEVICE_ATTR_RO(dsm_mask); 1234 1235 static ssize_t flags_show(struct device *dev, 1236 struct device_attribute *attr, char *buf) 1237 { 1238 u16 flags = to_nfit_memdev(dev)->flags; 1239 1240 return sprintf(buf, "%s%s%s%s%s\n", 1241 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", 1242 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", 1243 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", 1244 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", 1245 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : ""); 1246 } 1247 static DEVICE_ATTR_RO(flags); 1248 1249 static ssize_t id_show(struct device *dev, 1250 struct device_attribute *attr, char *buf) 1251 { 1252 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1253 1254 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) 1255 return sprintf(buf, "%04x-%02x-%04x-%08x\n", 1256 be16_to_cpu(dcr->vendor_id), 1257 dcr->manufacturing_location, 1258 be16_to_cpu(dcr->manufacturing_date), 1259 be32_to_cpu(dcr->serial_number)); 1260 else 1261 return sprintf(buf, "%04x-%08x\n", 1262 be16_to_cpu(dcr->vendor_id), 1263 be32_to_cpu(dcr->serial_number)); 1264 } 1265 static DEVICE_ATTR_RO(id); 1266 1267 static struct attribute *acpi_nfit_dimm_attributes[] = { 1268 &dev_attr_handle.attr, 1269 &dev_attr_phys_id.attr, 1270 &dev_attr_vendor.attr, 1271 &dev_attr_device.attr, 1272 &dev_attr_rev_id.attr, 1273 &dev_attr_subsystem_vendor.attr, 1274 &dev_attr_subsystem_device.attr, 1275 &dev_attr_subsystem_rev_id.attr, 1276 &dev_attr_format.attr, 1277 &dev_attr_formats.attr, 1278 &dev_attr_format1.attr, 1279 &dev_attr_serial.attr, 1280 &dev_attr_flags.attr, 1281 &dev_attr_id.attr, 1282 &dev_attr_family.attr, 1283 &dev_attr_dsm_mask.attr, 1284 NULL, 1285 }; 1286 1287 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, 1288 struct attribute *a, int n) 1289 { 1290 struct device *dev = container_of(kobj, struct device, kobj); 1291 struct nvdimm *nvdimm = to_nvdimm(dev); 1292 1293 if (!to_nfit_dcr(dev)) 1294 return 0; 1295 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) 1296 return 0; 1297 return a->mode; 1298 } 1299 1300 static struct attribute_group acpi_nfit_dimm_attribute_group = { 1301 .name = "nfit", 1302 .attrs = acpi_nfit_dimm_attributes, 1303 .is_visible = acpi_nfit_dimm_attr_visible, 1304 }; 1305 1306 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { 1307 &nvdimm_attribute_group, 1308 &nd_device_attribute_group, 1309 &acpi_nfit_dimm_attribute_group, 1310 NULL, 1311 }; 1312 1313 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, 1314 u32 device_handle) 1315 { 1316 struct nfit_mem *nfit_mem; 1317 1318 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1319 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) 1320 return nfit_mem->nvdimm; 1321 1322 return NULL; 1323 } 1324 1325 void __acpi_nvdimm_notify(struct device *dev, u32 event) 1326 { 1327 struct nfit_mem *nfit_mem; 1328 struct acpi_nfit_desc *acpi_desc; 1329 1330 dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__, 1331 event); 1332 1333 if (event != NFIT_NOTIFY_DIMM_HEALTH) { 1334 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev), 1335 event); 1336 return; 1337 } 1338 1339 acpi_desc = dev_get_drvdata(dev->parent); 1340 if (!acpi_desc) 1341 return; 1342 1343 /* 1344 * If we successfully retrieved acpi_desc, then we know nfit_mem data 1345 * is still valid. 1346 */ 1347 nfit_mem = dev_get_drvdata(dev); 1348 if (nfit_mem && nfit_mem->flags_attr) 1349 sysfs_notify_dirent(nfit_mem->flags_attr); 1350 } 1351 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify); 1352 1353 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data) 1354 { 1355 struct acpi_device *adev = data; 1356 struct device *dev = &adev->dev; 1357 1358 device_lock(dev->parent); 1359 __acpi_nvdimm_notify(dev, event); 1360 device_unlock(dev->parent); 1361 } 1362 1363 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, 1364 struct nfit_mem *nfit_mem, u32 device_handle) 1365 { 1366 struct acpi_device *adev, *adev_dimm; 1367 struct device *dev = acpi_desc->dev; 1368 unsigned long dsm_mask; 1369 const u8 *uuid; 1370 int i; 1371 1372 /* nfit test assumes 1:1 relationship between commands and dsms */ 1373 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; 1374 nfit_mem->family = NVDIMM_FAMILY_INTEL; 1375 adev = to_acpi_dev(acpi_desc); 1376 if (!adev) 1377 return 0; 1378 1379 adev_dimm = acpi_find_child_device(adev, device_handle, false); 1380 nfit_mem->adev = adev_dimm; 1381 if (!adev_dimm) { 1382 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", 1383 device_handle); 1384 return force_enable_dimms ? 0 : -ENODEV; 1385 } 1386 1387 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle, 1388 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) { 1389 dev_err(dev, "%s: notification registration failed\n", 1390 dev_name(&adev_dimm->dev)); 1391 return -ENXIO; 1392 } 1393 1394 /* 1395 * Until standardization materializes we need to consider 4 1396 * different command sets. Note, that checking for function0 (bit0) 1397 * tells us if any commands are reachable through this uuid. 1398 */ 1399 for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_MSFT; i++) 1400 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) 1401 break; 1402 1403 /* limit the supported commands to those that are publicly documented */ 1404 nfit_mem->family = i; 1405 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1406 dsm_mask = 0x3fe; 1407 if (disable_vendor_specific) 1408 dsm_mask &= ~(1 << ND_CMD_VENDOR); 1409 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { 1410 dsm_mask = 0x1c3c76; 1411 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { 1412 dsm_mask = 0x1fe; 1413 if (disable_vendor_specific) 1414 dsm_mask &= ~(1 << 8); 1415 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { 1416 dsm_mask = 0xffffffff; 1417 } else { 1418 dev_dbg(dev, "unknown dimm command family\n"); 1419 nfit_mem->family = -1; 1420 /* DSMs are optional, continue loading the driver... */ 1421 return 0; 1422 } 1423 1424 uuid = to_nfit_uuid(nfit_mem->family); 1425 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1426 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i)) 1427 set_bit(i, &nfit_mem->dsm_mask); 1428 1429 return 0; 1430 } 1431 1432 static void shutdown_dimm_notify(void *data) 1433 { 1434 struct acpi_nfit_desc *acpi_desc = data; 1435 struct nfit_mem *nfit_mem; 1436 1437 mutex_lock(&acpi_desc->init_mutex); 1438 /* 1439 * Clear out the nfit_mem->flags_attr and shut down dimm event 1440 * notifications. 1441 */ 1442 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1443 struct acpi_device *adev_dimm = nfit_mem->adev; 1444 1445 if (nfit_mem->flags_attr) { 1446 sysfs_put(nfit_mem->flags_attr); 1447 nfit_mem->flags_attr = NULL; 1448 } 1449 if (adev_dimm) 1450 acpi_remove_notify_handler(adev_dimm->handle, 1451 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify); 1452 } 1453 mutex_unlock(&acpi_desc->init_mutex); 1454 } 1455 1456 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) 1457 { 1458 struct nfit_mem *nfit_mem; 1459 int dimm_count = 0, rc; 1460 struct nvdimm *nvdimm; 1461 1462 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1463 struct acpi_nfit_flush_address *flush; 1464 unsigned long flags = 0, cmd_mask; 1465 u32 device_handle; 1466 u16 mem_flags; 1467 1468 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 1469 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); 1470 if (nvdimm) { 1471 dimm_count++; 1472 continue; 1473 } 1474 1475 if (nfit_mem->bdw && nfit_mem->memdev_pmem) 1476 flags |= NDD_ALIASING; 1477 1478 mem_flags = __to_nfit_memdev(nfit_mem)->flags; 1479 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) 1480 flags |= NDD_UNARMED; 1481 1482 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); 1483 if (rc) 1484 continue; 1485 1486 /* 1487 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL 1488 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the 1489 * userspace interface. 1490 */ 1491 cmd_mask = 1UL << ND_CMD_CALL; 1492 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) 1493 cmd_mask |= nfit_mem->dsm_mask; 1494 1495 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush 1496 : NULL; 1497 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, 1498 acpi_nfit_dimm_attribute_groups, 1499 flags, cmd_mask, flush ? flush->hint_count : 0, 1500 nfit_mem->flush_wpq); 1501 if (!nvdimm) 1502 return -ENOMEM; 1503 1504 nfit_mem->nvdimm = nvdimm; 1505 dimm_count++; 1506 1507 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) 1508 continue; 1509 1510 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n", 1511 nvdimm_name(nvdimm), 1512 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", 1513 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", 1514 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", 1515 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : ""); 1516 1517 } 1518 1519 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); 1520 if (rc) 1521 return rc; 1522 1523 /* 1524 * Now that dimms are successfully registered, and async registration 1525 * is flushed, attempt to enable event notification. 1526 */ 1527 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1528 struct kernfs_node *nfit_kernfs; 1529 1530 nvdimm = nfit_mem->nvdimm; 1531 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit"); 1532 if (nfit_kernfs) 1533 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs, 1534 "flags"); 1535 sysfs_put(nfit_kernfs); 1536 if (!nfit_mem->flags_attr) 1537 dev_warn(acpi_desc->dev, "%s: notifications disabled\n", 1538 nvdimm_name(nvdimm)); 1539 } 1540 1541 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify, 1542 acpi_desc); 1543 } 1544 1545 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) 1546 { 1547 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1548 const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS); 1549 struct acpi_device *adev; 1550 int i; 1551 1552 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; 1553 adev = to_acpi_dev(acpi_desc); 1554 if (!adev) 1555 return; 1556 1557 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) 1558 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i)) 1559 set_bit(i, &nd_desc->cmd_mask); 1560 } 1561 1562 static ssize_t range_index_show(struct device *dev, 1563 struct device_attribute *attr, char *buf) 1564 { 1565 struct nd_region *nd_region = to_nd_region(dev); 1566 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1567 1568 return sprintf(buf, "%d\n", nfit_spa->spa->range_index); 1569 } 1570 static DEVICE_ATTR_RO(range_index); 1571 1572 static struct attribute *acpi_nfit_region_attributes[] = { 1573 &dev_attr_range_index.attr, 1574 NULL, 1575 }; 1576 1577 static struct attribute_group acpi_nfit_region_attribute_group = { 1578 .name = "nfit", 1579 .attrs = acpi_nfit_region_attributes, 1580 }; 1581 1582 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { 1583 &nd_region_attribute_group, 1584 &nd_mapping_attribute_group, 1585 &nd_device_attribute_group, 1586 &nd_numa_attribute_group, 1587 &acpi_nfit_region_attribute_group, 1588 NULL, 1589 }; 1590 1591 /* enough info to uniquely specify an interleave set */ 1592 struct nfit_set_info { 1593 struct nfit_set_info_map { 1594 u64 region_offset; 1595 u32 serial_number; 1596 u32 pad; 1597 } mapping[0]; 1598 }; 1599 1600 static size_t sizeof_nfit_set_info(int num_mappings) 1601 { 1602 return sizeof(struct nfit_set_info) 1603 + num_mappings * sizeof(struct nfit_set_info_map); 1604 } 1605 1606 static int cmp_map_compat(const void *m0, const void *m1) 1607 { 1608 const struct nfit_set_info_map *map0 = m0; 1609 const struct nfit_set_info_map *map1 = m1; 1610 1611 return memcmp(&map0->region_offset, &map1->region_offset, 1612 sizeof(u64)); 1613 } 1614 1615 static int cmp_map(const void *m0, const void *m1) 1616 { 1617 const struct nfit_set_info_map *map0 = m0; 1618 const struct nfit_set_info_map *map1 = m1; 1619 1620 if (map0->region_offset < map1->region_offset) 1621 return -1; 1622 else if (map0->region_offset > map1->region_offset) 1623 return 1; 1624 return 0; 1625 } 1626 1627 /* Retrieve the nth entry referencing this spa */ 1628 static struct acpi_nfit_memory_map *memdev_from_spa( 1629 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) 1630 { 1631 struct nfit_memdev *nfit_memdev; 1632 1633 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) 1634 if (nfit_memdev->memdev->range_index == range_index) 1635 if (n-- == 0) 1636 return nfit_memdev->memdev; 1637 return NULL; 1638 } 1639 1640 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, 1641 struct nd_region_desc *ndr_desc, 1642 struct acpi_nfit_system_address *spa) 1643 { 1644 int i, spa_type = nfit_spa_type(spa); 1645 struct device *dev = acpi_desc->dev; 1646 struct nd_interleave_set *nd_set; 1647 u16 nr = ndr_desc->num_mappings; 1648 struct nfit_set_info *info; 1649 1650 if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE) 1651 /* pass */; 1652 else 1653 return 0; 1654 1655 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 1656 if (!nd_set) 1657 return -ENOMEM; 1658 1659 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); 1660 if (!info) 1661 return -ENOMEM; 1662 for (i = 0; i < nr; i++) { 1663 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 1664 struct nfit_set_info_map *map = &info->mapping[i]; 1665 struct nvdimm *nvdimm = mapping->nvdimm; 1666 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1667 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, 1668 spa->range_index, i); 1669 1670 if (!memdev || !nfit_mem->dcr) { 1671 dev_err(dev, "%s: failed to find DCR\n", __func__); 1672 return -ENODEV; 1673 } 1674 1675 map->region_offset = memdev->region_offset; 1676 map->serial_number = nfit_mem->dcr->serial_number; 1677 } 1678 1679 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 1680 cmp_map, NULL); 1681 nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 1682 1683 /* support namespaces created with the wrong sort order */ 1684 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 1685 cmp_map_compat, NULL); 1686 nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 1687 1688 ndr_desc->nd_set = nd_set; 1689 devm_kfree(dev, info); 1690 1691 return 0; 1692 } 1693 1694 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) 1695 { 1696 struct acpi_nfit_interleave *idt = mmio->idt; 1697 u32 sub_line_offset, line_index, line_offset; 1698 u64 line_no, table_skip_count, table_offset; 1699 1700 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); 1701 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); 1702 line_offset = idt->line_offset[line_index] 1703 * mmio->line_size; 1704 table_offset = table_skip_count * mmio->table_size; 1705 1706 return mmio->base_offset + line_offset + table_offset + sub_line_offset; 1707 } 1708 1709 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) 1710 { 1711 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 1712 u64 offset = nfit_blk->stat_offset + mmio->size * bw; 1713 const u32 STATUS_MASK = 0x80000037; 1714 1715 if (mmio->num_lines) 1716 offset = to_interleave_offset(offset, mmio); 1717 1718 return readl(mmio->addr.base + offset) & STATUS_MASK; 1719 } 1720 1721 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, 1722 resource_size_t dpa, unsigned int len, unsigned int write) 1723 { 1724 u64 cmd, offset; 1725 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 1726 1727 enum { 1728 BCW_OFFSET_MASK = (1ULL << 48)-1, 1729 BCW_LEN_SHIFT = 48, 1730 BCW_LEN_MASK = (1ULL << 8) - 1, 1731 BCW_CMD_SHIFT = 56, 1732 }; 1733 1734 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; 1735 len = len >> L1_CACHE_SHIFT; 1736 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; 1737 cmd |= ((u64) write) << BCW_CMD_SHIFT; 1738 1739 offset = nfit_blk->cmd_offset + mmio->size * bw; 1740 if (mmio->num_lines) 1741 offset = to_interleave_offset(offset, mmio); 1742 1743 writeq(cmd, mmio->addr.base + offset); 1744 nvdimm_flush(nfit_blk->nd_region); 1745 1746 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) 1747 readq(mmio->addr.base + offset); 1748 } 1749 1750 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 1751 resource_size_t dpa, void *iobuf, size_t len, int rw, 1752 unsigned int lane) 1753 { 1754 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 1755 unsigned int copied = 0; 1756 u64 base_offset; 1757 int rc; 1758 1759 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES 1760 + lane * mmio->size; 1761 write_blk_ctl(nfit_blk, lane, dpa, len, rw); 1762 while (len) { 1763 unsigned int c; 1764 u64 offset; 1765 1766 if (mmio->num_lines) { 1767 u32 line_offset; 1768 1769 offset = to_interleave_offset(base_offset + copied, 1770 mmio); 1771 div_u64_rem(offset, mmio->line_size, &line_offset); 1772 c = min_t(size_t, len, mmio->line_size - line_offset); 1773 } else { 1774 offset = base_offset + nfit_blk->bdw_offset; 1775 c = len; 1776 } 1777 1778 if (rw) 1779 memcpy_to_pmem(mmio->addr.aperture + offset, 1780 iobuf + copied, c); 1781 else { 1782 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) 1783 mmio_flush_range((void __force *) 1784 mmio->addr.aperture + offset, c); 1785 1786 memcpy_from_pmem(iobuf + copied, 1787 mmio->addr.aperture + offset, c); 1788 } 1789 1790 copied += c; 1791 len -= c; 1792 } 1793 1794 if (rw) 1795 nvdimm_flush(nfit_blk->nd_region); 1796 1797 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; 1798 return rc; 1799 } 1800 1801 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, 1802 resource_size_t dpa, void *iobuf, u64 len, int rw) 1803 { 1804 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); 1805 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 1806 struct nd_region *nd_region = nfit_blk->nd_region; 1807 unsigned int lane, copied = 0; 1808 int rc = 0; 1809 1810 lane = nd_region_acquire_lane(nd_region); 1811 while (len) { 1812 u64 c = min(len, mmio->size); 1813 1814 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, 1815 iobuf + copied, c, rw, lane); 1816 if (rc) 1817 break; 1818 1819 copied += c; 1820 len -= c; 1821 } 1822 nd_region_release_lane(nd_region, lane); 1823 1824 return rc; 1825 } 1826 1827 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, 1828 struct acpi_nfit_interleave *idt, u16 interleave_ways) 1829 { 1830 if (idt) { 1831 mmio->num_lines = idt->line_count; 1832 mmio->line_size = idt->line_size; 1833 if (interleave_ways == 0) 1834 return -ENXIO; 1835 mmio->table_size = mmio->num_lines * interleave_ways 1836 * mmio->line_size; 1837 } 1838 1839 return 0; 1840 } 1841 1842 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, 1843 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) 1844 { 1845 struct nd_cmd_dimm_flags flags; 1846 int rc; 1847 1848 memset(&flags, 0, sizeof(flags)); 1849 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, 1850 sizeof(flags), NULL); 1851 1852 if (rc >= 0 && flags.status == 0) 1853 nfit_blk->dimm_flags = flags.flags; 1854 else if (rc == -ENOTTY) { 1855 /* fall back to a conservative default */ 1856 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; 1857 rc = 0; 1858 } else 1859 rc = -ENXIO; 1860 1861 return rc; 1862 } 1863 1864 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, 1865 struct device *dev) 1866 { 1867 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1868 struct nd_blk_region *ndbr = to_nd_blk_region(dev); 1869 struct nfit_blk_mmio *mmio; 1870 struct nfit_blk *nfit_blk; 1871 struct nfit_mem *nfit_mem; 1872 struct nvdimm *nvdimm; 1873 int rc; 1874 1875 nvdimm = nd_blk_region_to_dimm(ndbr); 1876 nfit_mem = nvdimm_provider_data(nvdimm); 1877 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { 1878 dev_dbg(dev, "%s: missing%s%s%s\n", __func__, 1879 nfit_mem ? "" : " nfit_mem", 1880 (nfit_mem && nfit_mem->dcr) ? "" : " dcr", 1881 (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); 1882 return -ENXIO; 1883 } 1884 1885 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); 1886 if (!nfit_blk) 1887 return -ENOMEM; 1888 nd_blk_region_set_provider_data(ndbr, nfit_blk); 1889 nfit_blk->nd_region = to_nd_region(dev); 1890 1891 /* map block aperture memory */ 1892 nfit_blk->bdw_offset = nfit_mem->bdw->offset; 1893 mmio = &nfit_blk->mmio[BDW]; 1894 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, 1895 nfit_mem->spa_bdw->length, ARCH_MEMREMAP_PMEM); 1896 if (!mmio->addr.base) { 1897 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, 1898 nvdimm_name(nvdimm)); 1899 return -ENOMEM; 1900 } 1901 mmio->size = nfit_mem->bdw->size; 1902 mmio->base_offset = nfit_mem->memdev_bdw->region_offset; 1903 mmio->idt = nfit_mem->idt_bdw; 1904 mmio->spa = nfit_mem->spa_bdw; 1905 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, 1906 nfit_mem->memdev_bdw->interleave_ways); 1907 if (rc) { 1908 dev_dbg(dev, "%s: %s failed to init bdw interleave\n", 1909 __func__, nvdimm_name(nvdimm)); 1910 return rc; 1911 } 1912 1913 /* map block control memory */ 1914 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; 1915 nfit_blk->stat_offset = nfit_mem->dcr->status_offset; 1916 mmio = &nfit_blk->mmio[DCR]; 1917 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, 1918 nfit_mem->spa_dcr->length); 1919 if (!mmio->addr.base) { 1920 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, 1921 nvdimm_name(nvdimm)); 1922 return -ENOMEM; 1923 } 1924 mmio->size = nfit_mem->dcr->window_size; 1925 mmio->base_offset = nfit_mem->memdev_dcr->region_offset; 1926 mmio->idt = nfit_mem->idt_dcr; 1927 mmio->spa = nfit_mem->spa_dcr; 1928 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, 1929 nfit_mem->memdev_dcr->interleave_ways); 1930 if (rc) { 1931 dev_dbg(dev, "%s: %s failed to init dcr interleave\n", 1932 __func__, nvdimm_name(nvdimm)); 1933 return rc; 1934 } 1935 1936 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); 1937 if (rc < 0) { 1938 dev_dbg(dev, "%s: %s failed get DIMM flags\n", 1939 __func__, nvdimm_name(nvdimm)); 1940 return rc; 1941 } 1942 1943 if (nvdimm_has_flush(nfit_blk->nd_region) < 0) 1944 dev_warn(dev, "unable to guarantee persistence of writes\n"); 1945 1946 if (mmio->line_size == 0) 1947 return 0; 1948 1949 if ((u32) nfit_blk->cmd_offset % mmio->line_size 1950 + 8 > mmio->line_size) { 1951 dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); 1952 return -ENXIO; 1953 } else if ((u32) nfit_blk->stat_offset % mmio->line_size 1954 + 8 > mmio->line_size) { 1955 dev_dbg(dev, "stat_offset crosses interleave boundary\n"); 1956 return -ENXIO; 1957 } 1958 1959 return 0; 1960 } 1961 1962 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, 1963 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) 1964 { 1965 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1966 struct acpi_nfit_system_address *spa = nfit_spa->spa; 1967 int cmd_rc, rc; 1968 1969 cmd->address = spa->address; 1970 cmd->length = spa->length; 1971 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, 1972 sizeof(*cmd), &cmd_rc); 1973 if (rc < 0) 1974 return rc; 1975 return cmd_rc; 1976 } 1977 1978 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) 1979 { 1980 int rc; 1981 int cmd_rc; 1982 struct nd_cmd_ars_start ars_start; 1983 struct acpi_nfit_system_address *spa = nfit_spa->spa; 1984 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1985 1986 memset(&ars_start, 0, sizeof(ars_start)); 1987 ars_start.address = spa->address; 1988 ars_start.length = spa->length; 1989 if (nfit_spa_type(spa) == NFIT_SPA_PM) 1990 ars_start.type = ND_ARS_PERSISTENT; 1991 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) 1992 ars_start.type = ND_ARS_VOLATILE; 1993 else 1994 return -ENOTTY; 1995 1996 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 1997 sizeof(ars_start), &cmd_rc); 1998 1999 if (rc < 0) 2000 return rc; 2001 return cmd_rc; 2002 } 2003 2004 static int ars_continue(struct acpi_nfit_desc *acpi_desc) 2005 { 2006 int rc, cmd_rc; 2007 struct nd_cmd_ars_start ars_start; 2008 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2009 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2010 2011 memset(&ars_start, 0, sizeof(ars_start)); 2012 ars_start.address = ars_status->restart_address; 2013 ars_start.length = ars_status->restart_length; 2014 ars_start.type = ars_status->type; 2015 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2016 sizeof(ars_start), &cmd_rc); 2017 if (rc < 0) 2018 return rc; 2019 return cmd_rc; 2020 } 2021 2022 static int ars_get_status(struct acpi_nfit_desc *acpi_desc) 2023 { 2024 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2025 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2026 int rc, cmd_rc; 2027 2028 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, 2029 acpi_desc->ars_status_size, &cmd_rc); 2030 if (rc < 0) 2031 return rc; 2032 return cmd_rc; 2033 } 2034 2035 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc, 2036 struct nd_cmd_ars_status *ars_status) 2037 { 2038 struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus; 2039 int rc; 2040 u32 i; 2041 2042 /* 2043 * First record starts at 44 byte offset from the start of the 2044 * payload. 2045 */ 2046 if (ars_status->out_length < 44) 2047 return 0; 2048 for (i = 0; i < ars_status->num_records; i++) { 2049 /* only process full records */ 2050 if (ars_status->out_length 2051 < 44 + sizeof(struct nd_ars_record) * (i + 1)) 2052 break; 2053 rc = nvdimm_bus_add_poison(nvdimm_bus, 2054 ars_status->records[i].err_address, 2055 ars_status->records[i].length); 2056 if (rc) 2057 return rc; 2058 } 2059 if (i < ars_status->num_records) 2060 dev_warn(acpi_desc->dev, "detected truncated ars results\n"); 2061 2062 return 0; 2063 } 2064 2065 static void acpi_nfit_remove_resource(void *data) 2066 { 2067 struct resource *res = data; 2068 2069 remove_resource(res); 2070 } 2071 2072 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, 2073 struct nd_region_desc *ndr_desc) 2074 { 2075 struct resource *res, *nd_res = ndr_desc->res; 2076 int is_pmem, ret; 2077 2078 /* No operation if the region is already registered as PMEM */ 2079 is_pmem = region_intersects(nd_res->start, resource_size(nd_res), 2080 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); 2081 if (is_pmem == REGION_INTERSECTS) 2082 return 0; 2083 2084 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); 2085 if (!res) 2086 return -ENOMEM; 2087 2088 res->name = "Persistent Memory"; 2089 res->start = nd_res->start; 2090 res->end = nd_res->end; 2091 res->flags = IORESOURCE_MEM; 2092 res->desc = IORES_DESC_PERSISTENT_MEMORY; 2093 2094 ret = insert_resource(&iomem_resource, res); 2095 if (ret) 2096 return ret; 2097 2098 ret = devm_add_action_or_reset(acpi_desc->dev, 2099 acpi_nfit_remove_resource, 2100 res); 2101 if (ret) 2102 return ret; 2103 2104 return 0; 2105 } 2106 2107 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, 2108 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc, 2109 struct acpi_nfit_memory_map *memdev, 2110 struct nfit_spa *nfit_spa) 2111 { 2112 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, 2113 memdev->device_handle); 2114 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2115 struct nd_blk_region_desc *ndbr_desc; 2116 struct nfit_mem *nfit_mem; 2117 int blk_valid = 0; 2118 2119 if (!nvdimm) { 2120 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", 2121 spa->range_index, memdev->device_handle); 2122 return -ENODEV; 2123 } 2124 2125 mapping->nvdimm = nvdimm; 2126 switch (nfit_spa_type(spa)) { 2127 case NFIT_SPA_PM: 2128 case NFIT_SPA_VOLATILE: 2129 mapping->start = memdev->address; 2130 mapping->size = memdev->region_size; 2131 break; 2132 case NFIT_SPA_DCR: 2133 nfit_mem = nvdimm_provider_data(nvdimm); 2134 if (!nfit_mem || !nfit_mem->bdw) { 2135 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", 2136 spa->range_index, nvdimm_name(nvdimm)); 2137 } else { 2138 mapping->size = nfit_mem->bdw->capacity; 2139 mapping->start = nfit_mem->bdw->start_address; 2140 ndr_desc->num_lanes = nfit_mem->bdw->windows; 2141 blk_valid = 1; 2142 } 2143 2144 ndr_desc->mapping = mapping; 2145 ndr_desc->num_mappings = blk_valid; 2146 ndbr_desc = to_blk_region_desc(ndr_desc); 2147 ndbr_desc->enable = acpi_nfit_blk_region_enable; 2148 ndbr_desc->do_io = acpi_desc->blk_do_io; 2149 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, 2150 ndr_desc); 2151 if (!nfit_spa->nd_region) 2152 return -ENOMEM; 2153 break; 2154 } 2155 2156 return 0; 2157 } 2158 2159 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) 2160 { 2161 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2162 nfit_spa_type(spa) == NFIT_SPA_VCD || 2163 nfit_spa_type(spa) == NFIT_SPA_PDISK || 2164 nfit_spa_type(spa) == NFIT_SPA_PCD); 2165 } 2166 2167 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, 2168 struct nfit_spa *nfit_spa) 2169 { 2170 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS]; 2171 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2172 struct nd_blk_region_desc ndbr_desc; 2173 struct nd_region_desc *ndr_desc; 2174 struct nfit_memdev *nfit_memdev; 2175 struct nvdimm_bus *nvdimm_bus; 2176 struct resource res; 2177 int count = 0, rc; 2178 2179 if (nfit_spa->nd_region) 2180 return 0; 2181 2182 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { 2183 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n", 2184 __func__); 2185 return 0; 2186 } 2187 2188 memset(&res, 0, sizeof(res)); 2189 memset(&mappings, 0, sizeof(mappings)); 2190 memset(&ndbr_desc, 0, sizeof(ndbr_desc)); 2191 res.start = spa->address; 2192 res.end = res.start + spa->length - 1; 2193 ndr_desc = &ndbr_desc.ndr_desc; 2194 ndr_desc->res = &res; 2195 ndr_desc->provider_data = nfit_spa; 2196 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; 2197 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) 2198 ndr_desc->numa_node = acpi_map_pxm_to_online_node( 2199 spa->proximity_domain); 2200 else 2201 ndr_desc->numa_node = NUMA_NO_NODE; 2202 2203 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 2204 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 2205 struct nd_mapping_desc *mapping; 2206 2207 if (memdev->range_index != spa->range_index) 2208 continue; 2209 if (count >= ND_MAX_MAPPINGS) { 2210 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", 2211 spa->range_index, ND_MAX_MAPPINGS); 2212 return -ENXIO; 2213 } 2214 mapping = &mappings[count++]; 2215 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc, 2216 memdev, nfit_spa); 2217 if (rc) 2218 goto out; 2219 } 2220 2221 ndr_desc->mapping = mappings; 2222 ndr_desc->num_mappings = count; 2223 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2224 if (rc) 2225 goto out; 2226 2227 nvdimm_bus = acpi_desc->nvdimm_bus; 2228 if (nfit_spa_type(spa) == NFIT_SPA_PM) { 2229 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); 2230 if (rc) { 2231 dev_warn(acpi_desc->dev, 2232 "failed to insert pmem resource to iomem: %d\n", 2233 rc); 2234 goto out; 2235 } 2236 2237 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2238 ndr_desc); 2239 if (!nfit_spa->nd_region) 2240 rc = -ENOMEM; 2241 } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) { 2242 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, 2243 ndr_desc); 2244 if (!nfit_spa->nd_region) 2245 rc = -ENOMEM; 2246 } else if (nfit_spa_is_virtual(spa)) { 2247 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2248 ndr_desc); 2249 if (!nfit_spa->nd_region) 2250 rc = -ENOMEM; 2251 } 2252 2253 out: 2254 if (rc) 2255 dev_err(acpi_desc->dev, "failed to register spa range %d\n", 2256 nfit_spa->spa->range_index); 2257 return rc; 2258 } 2259 2260 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc, 2261 u32 max_ars) 2262 { 2263 struct device *dev = acpi_desc->dev; 2264 struct nd_cmd_ars_status *ars_status; 2265 2266 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) { 2267 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size); 2268 return 0; 2269 } 2270 2271 if (acpi_desc->ars_status) 2272 devm_kfree(dev, acpi_desc->ars_status); 2273 acpi_desc->ars_status = NULL; 2274 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL); 2275 if (!ars_status) 2276 return -ENOMEM; 2277 acpi_desc->ars_status = ars_status; 2278 acpi_desc->ars_status_size = max_ars; 2279 return 0; 2280 } 2281 2282 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc, 2283 struct nfit_spa *nfit_spa) 2284 { 2285 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2286 int rc; 2287 2288 if (!nfit_spa->max_ars) { 2289 struct nd_cmd_ars_cap ars_cap; 2290 2291 memset(&ars_cap, 0, sizeof(ars_cap)); 2292 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); 2293 if (rc < 0) 2294 return rc; 2295 nfit_spa->max_ars = ars_cap.max_ars_out; 2296 nfit_spa->clear_err_unit = ars_cap.clear_err_unit; 2297 /* check that the supported scrub types match the spa type */ 2298 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE && 2299 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0) 2300 return -ENOTTY; 2301 else if (nfit_spa_type(spa) == NFIT_SPA_PM && 2302 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0) 2303 return -ENOTTY; 2304 } 2305 2306 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars)) 2307 return -ENOMEM; 2308 2309 rc = ars_get_status(acpi_desc); 2310 if (rc < 0 && rc != -ENOSPC) 2311 return rc; 2312 2313 if (ars_status_process_records(acpi_desc, acpi_desc->ars_status)) 2314 return -ENOMEM; 2315 2316 return 0; 2317 } 2318 2319 static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, 2320 struct nfit_spa *nfit_spa) 2321 { 2322 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2323 unsigned int overflow_retry = scrub_overflow_abort; 2324 u64 init_ars_start = 0, init_ars_len = 0; 2325 struct device *dev = acpi_desc->dev; 2326 unsigned int tmo = scrub_timeout; 2327 int rc; 2328 2329 if (!nfit_spa->ars_required || !nfit_spa->nd_region) 2330 return; 2331 2332 rc = ars_start(acpi_desc, nfit_spa); 2333 /* 2334 * If we timed out the initial scan we'll still be busy here, 2335 * and will wait another timeout before giving up permanently. 2336 */ 2337 if (rc < 0 && rc != -EBUSY) 2338 return; 2339 2340 do { 2341 u64 ars_start, ars_len; 2342 2343 if (acpi_desc->cancel) 2344 break; 2345 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2346 if (rc == -ENOTTY) 2347 break; 2348 if (rc == -EBUSY && !tmo) { 2349 dev_warn(dev, "range %d ars timeout, aborting\n", 2350 spa->range_index); 2351 break; 2352 } 2353 2354 if (rc == -EBUSY) { 2355 /* 2356 * Note, entries may be appended to the list 2357 * while the lock is dropped, but the workqueue 2358 * being active prevents entries being deleted / 2359 * freed. 2360 */ 2361 mutex_unlock(&acpi_desc->init_mutex); 2362 ssleep(1); 2363 tmo--; 2364 mutex_lock(&acpi_desc->init_mutex); 2365 continue; 2366 } 2367 2368 /* we got some results, but there are more pending... */ 2369 if (rc == -ENOSPC && overflow_retry--) { 2370 if (!init_ars_len) { 2371 init_ars_len = acpi_desc->ars_status->length; 2372 init_ars_start = acpi_desc->ars_status->address; 2373 } 2374 rc = ars_continue(acpi_desc); 2375 } 2376 2377 if (rc < 0) { 2378 dev_warn(dev, "range %d ars continuation failed\n", 2379 spa->range_index); 2380 break; 2381 } 2382 2383 if (init_ars_len) { 2384 ars_start = init_ars_start; 2385 ars_len = init_ars_len; 2386 } else { 2387 ars_start = acpi_desc->ars_status->address; 2388 ars_len = acpi_desc->ars_status->length; 2389 } 2390 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n", 2391 spa->range_index, ars_start, ars_len); 2392 /* notify the region about new poison entries */ 2393 nvdimm_region_notify(nfit_spa->nd_region, 2394 NVDIMM_REVALIDATE_POISON); 2395 break; 2396 } while (1); 2397 } 2398 2399 static void acpi_nfit_scrub(struct work_struct *work) 2400 { 2401 struct device *dev; 2402 u64 init_scrub_length = 0; 2403 struct nfit_spa *nfit_spa; 2404 u64 init_scrub_address = 0; 2405 bool init_ars_done = false; 2406 struct acpi_nfit_desc *acpi_desc; 2407 unsigned int tmo = scrub_timeout; 2408 unsigned int overflow_retry = scrub_overflow_abort; 2409 2410 acpi_desc = container_of(work, typeof(*acpi_desc), work); 2411 dev = acpi_desc->dev; 2412 2413 /* 2414 * We scrub in 2 phases. The first phase waits for any platform 2415 * firmware initiated scrubs to complete and then we go search for the 2416 * affected spa regions to mark them scanned. In the second phase we 2417 * initiate a directed scrub for every range that was not scrubbed in 2418 * phase 1. If we're called for a 'rescan', we harmlessly pass through 2419 * the first phase, but really only care about running phase 2, where 2420 * regions can be notified of new poison. 2421 */ 2422 2423 /* process platform firmware initiated scrubs */ 2424 retry: 2425 mutex_lock(&acpi_desc->init_mutex); 2426 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2427 struct nd_cmd_ars_status *ars_status; 2428 struct acpi_nfit_system_address *spa; 2429 u64 ars_start, ars_len; 2430 int rc; 2431 2432 if (acpi_desc->cancel) 2433 break; 2434 2435 if (nfit_spa->nd_region) 2436 continue; 2437 2438 if (init_ars_done) { 2439 /* 2440 * No need to re-query, we're now just 2441 * reconciling all the ranges covered by the 2442 * initial scrub 2443 */ 2444 rc = 0; 2445 } else 2446 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2447 2448 if (rc == -ENOTTY) { 2449 /* no ars capability, just register spa and move on */ 2450 acpi_nfit_register_region(acpi_desc, nfit_spa); 2451 continue; 2452 } 2453 2454 if (rc == -EBUSY && !tmo) { 2455 /* fallthrough to directed scrub in phase 2 */ 2456 dev_warn(dev, "timeout awaiting ars results, continuing...\n"); 2457 break; 2458 } else if (rc == -EBUSY) { 2459 mutex_unlock(&acpi_desc->init_mutex); 2460 ssleep(1); 2461 tmo--; 2462 goto retry; 2463 } 2464 2465 /* we got some results, but there are more pending... */ 2466 if (rc == -ENOSPC && overflow_retry--) { 2467 ars_status = acpi_desc->ars_status; 2468 /* 2469 * Record the original scrub range, so that we 2470 * can recall all the ranges impacted by the 2471 * initial scrub. 2472 */ 2473 if (!init_scrub_length) { 2474 init_scrub_length = ars_status->length; 2475 init_scrub_address = ars_status->address; 2476 } 2477 rc = ars_continue(acpi_desc); 2478 if (rc == 0) { 2479 mutex_unlock(&acpi_desc->init_mutex); 2480 goto retry; 2481 } 2482 } 2483 2484 if (rc < 0) { 2485 /* 2486 * Initial scrub failed, we'll give it one more 2487 * try below... 2488 */ 2489 break; 2490 } 2491 2492 /* We got some final results, record completed ranges */ 2493 ars_status = acpi_desc->ars_status; 2494 if (init_scrub_length) { 2495 ars_start = init_scrub_address; 2496 ars_len = ars_start + init_scrub_length; 2497 } else { 2498 ars_start = ars_status->address; 2499 ars_len = ars_status->length; 2500 } 2501 spa = nfit_spa->spa; 2502 2503 if (!init_ars_done) { 2504 init_ars_done = true; 2505 dev_dbg(dev, "init scrub %#llx + %#llx complete\n", 2506 ars_start, ars_len); 2507 } 2508 if (ars_start <= spa->address && ars_start + ars_len 2509 >= spa->address + spa->length) 2510 acpi_nfit_register_region(acpi_desc, nfit_spa); 2511 } 2512 2513 /* 2514 * For all the ranges not covered by an initial scrub we still 2515 * want to see if there are errors, but it's ok to discover them 2516 * asynchronously. 2517 */ 2518 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2519 /* 2520 * Flag all the ranges that still need scrubbing, but 2521 * register them now to make data available. 2522 */ 2523 if (!nfit_spa->nd_region) { 2524 nfit_spa->ars_required = 1; 2525 acpi_nfit_register_region(acpi_desc, nfit_spa); 2526 } 2527 } 2528 2529 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2530 acpi_nfit_async_scrub(acpi_desc, nfit_spa); 2531 acpi_desc->scrub_count++; 2532 if (acpi_desc->scrub_count_state) 2533 sysfs_notify_dirent(acpi_desc->scrub_count_state); 2534 mutex_unlock(&acpi_desc->init_mutex); 2535 } 2536 2537 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) 2538 { 2539 struct nfit_spa *nfit_spa; 2540 int rc; 2541 2542 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2543 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) { 2544 /* BLK regions don't need to wait for ars results */ 2545 rc = acpi_nfit_register_region(acpi_desc, nfit_spa); 2546 if (rc) 2547 return rc; 2548 } 2549 2550 queue_work(nfit_wq, &acpi_desc->work); 2551 return 0; 2552 } 2553 2554 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, 2555 struct nfit_table_prev *prev) 2556 { 2557 struct device *dev = acpi_desc->dev; 2558 2559 if (!list_empty(&prev->spas) || 2560 !list_empty(&prev->memdevs) || 2561 !list_empty(&prev->dcrs) || 2562 !list_empty(&prev->bdws) || 2563 !list_empty(&prev->idts) || 2564 !list_empty(&prev->flushes)) { 2565 dev_err(dev, "new nfit deletes entries (unsupported)\n"); 2566 return -ENXIO; 2567 } 2568 return 0; 2569 } 2570 2571 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) 2572 { 2573 struct device *dev = acpi_desc->dev; 2574 struct kernfs_node *nfit; 2575 struct device *bus_dev; 2576 2577 if (!ars_supported(acpi_desc->nvdimm_bus)) 2578 return 0; 2579 2580 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 2581 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); 2582 if (!nfit) { 2583 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); 2584 return -ENODEV; 2585 } 2586 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); 2587 sysfs_put(nfit); 2588 if (!acpi_desc->scrub_count_state) { 2589 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); 2590 return -ENODEV; 2591 } 2592 2593 return 0; 2594 } 2595 2596 static void acpi_nfit_destruct(void *data) 2597 { 2598 struct acpi_nfit_desc *acpi_desc = data; 2599 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 2600 2601 /* 2602 * Destruct under acpi_desc_lock so that nfit_handle_mce does not 2603 * race teardown 2604 */ 2605 mutex_lock(&acpi_desc_lock); 2606 acpi_desc->cancel = 1; 2607 /* 2608 * Bounce the nvdimm bus lock to make sure any in-flight 2609 * acpi_nfit_ars_rescan() submissions have had a chance to 2610 * either submit or see ->cancel set. 2611 */ 2612 device_lock(bus_dev); 2613 device_unlock(bus_dev); 2614 2615 flush_workqueue(nfit_wq); 2616 if (acpi_desc->scrub_count_state) 2617 sysfs_put(acpi_desc->scrub_count_state); 2618 nvdimm_bus_unregister(acpi_desc->nvdimm_bus); 2619 acpi_desc->nvdimm_bus = NULL; 2620 list_del(&acpi_desc->list); 2621 mutex_unlock(&acpi_desc_lock); 2622 } 2623 2624 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) 2625 { 2626 struct device *dev = acpi_desc->dev; 2627 struct nfit_table_prev prev; 2628 const void *end; 2629 int rc; 2630 2631 if (!acpi_desc->nvdimm_bus) { 2632 acpi_nfit_init_dsms(acpi_desc); 2633 2634 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, 2635 &acpi_desc->nd_desc); 2636 if (!acpi_desc->nvdimm_bus) 2637 return -ENOMEM; 2638 2639 rc = devm_add_action_or_reset(dev, acpi_nfit_destruct, 2640 acpi_desc); 2641 if (rc) 2642 return rc; 2643 2644 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); 2645 if (rc) 2646 return rc; 2647 2648 /* register this acpi_desc for mce notifications */ 2649 mutex_lock(&acpi_desc_lock); 2650 list_add_tail(&acpi_desc->list, &acpi_descs); 2651 mutex_unlock(&acpi_desc_lock); 2652 } 2653 2654 mutex_lock(&acpi_desc->init_mutex); 2655 2656 INIT_LIST_HEAD(&prev.spas); 2657 INIT_LIST_HEAD(&prev.memdevs); 2658 INIT_LIST_HEAD(&prev.dcrs); 2659 INIT_LIST_HEAD(&prev.bdws); 2660 INIT_LIST_HEAD(&prev.idts); 2661 INIT_LIST_HEAD(&prev.flushes); 2662 2663 list_cut_position(&prev.spas, &acpi_desc->spas, 2664 acpi_desc->spas.prev); 2665 list_cut_position(&prev.memdevs, &acpi_desc->memdevs, 2666 acpi_desc->memdevs.prev); 2667 list_cut_position(&prev.dcrs, &acpi_desc->dcrs, 2668 acpi_desc->dcrs.prev); 2669 list_cut_position(&prev.bdws, &acpi_desc->bdws, 2670 acpi_desc->bdws.prev); 2671 list_cut_position(&prev.idts, &acpi_desc->idts, 2672 acpi_desc->idts.prev); 2673 list_cut_position(&prev.flushes, &acpi_desc->flushes, 2674 acpi_desc->flushes.prev); 2675 2676 end = data + sz; 2677 while (!IS_ERR_OR_NULL(data)) 2678 data = add_table(acpi_desc, &prev, data, end); 2679 2680 if (IS_ERR(data)) { 2681 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__, 2682 PTR_ERR(data)); 2683 rc = PTR_ERR(data); 2684 goto out_unlock; 2685 } 2686 2687 rc = acpi_nfit_check_deletions(acpi_desc, &prev); 2688 if (rc) 2689 goto out_unlock; 2690 2691 rc = nfit_mem_init(acpi_desc); 2692 if (rc) 2693 goto out_unlock; 2694 2695 rc = acpi_nfit_register_dimms(acpi_desc); 2696 if (rc) 2697 goto out_unlock; 2698 2699 rc = acpi_nfit_register_regions(acpi_desc); 2700 2701 out_unlock: 2702 mutex_unlock(&acpi_desc->init_mutex); 2703 return rc; 2704 } 2705 EXPORT_SYMBOL_GPL(acpi_nfit_init); 2706 2707 struct acpi_nfit_flush_work { 2708 struct work_struct work; 2709 struct completion cmp; 2710 }; 2711 2712 static void flush_probe(struct work_struct *work) 2713 { 2714 struct acpi_nfit_flush_work *flush; 2715 2716 flush = container_of(work, typeof(*flush), work); 2717 complete(&flush->cmp); 2718 } 2719 2720 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) 2721 { 2722 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 2723 struct device *dev = acpi_desc->dev; 2724 struct acpi_nfit_flush_work flush; 2725 int rc; 2726 2727 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ 2728 device_lock(dev); 2729 device_unlock(dev); 2730 2731 /* 2732 * Scrub work could take 10s of seconds, userspace may give up so we 2733 * need to be interruptible while waiting. 2734 */ 2735 INIT_WORK_ONSTACK(&flush.work, flush_probe); 2736 COMPLETION_INITIALIZER_ONSTACK(flush.cmp); 2737 queue_work(nfit_wq, &flush.work); 2738 2739 rc = wait_for_completion_interruptible(&flush.cmp); 2740 cancel_work_sync(&flush.work); 2741 return rc; 2742 } 2743 2744 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, 2745 struct nvdimm *nvdimm, unsigned int cmd) 2746 { 2747 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 2748 2749 if (nvdimm) 2750 return 0; 2751 if (cmd != ND_CMD_ARS_START) 2752 return 0; 2753 2754 /* 2755 * The kernel and userspace may race to initiate a scrub, but 2756 * the scrub thread is prepared to lose that initial race. It 2757 * just needs guarantees that any ars it initiates are not 2758 * interrupted by any intervening start reqeusts from userspace. 2759 */ 2760 if (work_busy(&acpi_desc->work)) 2761 return -EBUSY; 2762 2763 return 0; 2764 } 2765 2766 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc) 2767 { 2768 struct device *dev = acpi_desc->dev; 2769 struct nfit_spa *nfit_spa; 2770 2771 if (work_busy(&acpi_desc->work)) 2772 return -EBUSY; 2773 2774 if (acpi_desc->cancel) 2775 return 0; 2776 2777 mutex_lock(&acpi_desc->init_mutex); 2778 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2779 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2780 2781 if (nfit_spa_type(spa) != NFIT_SPA_PM) 2782 continue; 2783 2784 nfit_spa->ars_required = 1; 2785 } 2786 queue_work(nfit_wq, &acpi_desc->work); 2787 dev_dbg(dev, "%s: ars_scan triggered\n", __func__); 2788 mutex_unlock(&acpi_desc->init_mutex); 2789 2790 return 0; 2791 } 2792 2793 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) 2794 { 2795 struct nvdimm_bus_descriptor *nd_desc; 2796 2797 dev_set_drvdata(dev, acpi_desc); 2798 acpi_desc->dev = dev; 2799 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; 2800 nd_desc = &acpi_desc->nd_desc; 2801 nd_desc->provider_name = "ACPI.NFIT"; 2802 nd_desc->module = THIS_MODULE; 2803 nd_desc->ndctl = acpi_nfit_ctl; 2804 nd_desc->flush_probe = acpi_nfit_flush_probe; 2805 nd_desc->clear_to_send = acpi_nfit_clear_to_send; 2806 nd_desc->attr_groups = acpi_nfit_attribute_groups; 2807 2808 INIT_LIST_HEAD(&acpi_desc->spas); 2809 INIT_LIST_HEAD(&acpi_desc->dcrs); 2810 INIT_LIST_HEAD(&acpi_desc->bdws); 2811 INIT_LIST_HEAD(&acpi_desc->idts); 2812 INIT_LIST_HEAD(&acpi_desc->flushes); 2813 INIT_LIST_HEAD(&acpi_desc->memdevs); 2814 INIT_LIST_HEAD(&acpi_desc->dimms); 2815 INIT_LIST_HEAD(&acpi_desc->list); 2816 mutex_init(&acpi_desc->init_mutex); 2817 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); 2818 } 2819 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); 2820 2821 static int acpi_nfit_add(struct acpi_device *adev) 2822 { 2823 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 2824 struct acpi_nfit_desc *acpi_desc; 2825 struct device *dev = &adev->dev; 2826 struct acpi_table_header *tbl; 2827 acpi_status status = AE_OK; 2828 acpi_size sz; 2829 int rc = 0; 2830 2831 status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl); 2832 if (ACPI_FAILURE(status)) { 2833 /* This is ok, we could have an nvdimm hotplugged later */ 2834 dev_dbg(dev, "failed to find NFIT at startup\n"); 2835 return 0; 2836 } 2837 sz = tbl->length; 2838 2839 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 2840 if (!acpi_desc) 2841 return -ENOMEM; 2842 acpi_nfit_desc_init(acpi_desc, &adev->dev); 2843 2844 /* Save the acpi header for exporting the revision via sysfs */ 2845 acpi_desc->acpi_header = *tbl; 2846 2847 /* Evaluate _FIT and override with that if present */ 2848 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); 2849 if (ACPI_SUCCESS(status) && buf.length > 0) { 2850 union acpi_object *obj = buf.pointer; 2851 2852 if (obj->type == ACPI_TYPE_BUFFER) 2853 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 2854 obj->buffer.length); 2855 else 2856 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n", 2857 __func__, (int) obj->type); 2858 kfree(buf.pointer); 2859 } else 2860 /* skip over the lead-in header table */ 2861 rc = acpi_nfit_init(acpi_desc, (void *) tbl 2862 + sizeof(struct acpi_table_nfit), 2863 sz - sizeof(struct acpi_table_nfit)); 2864 return rc; 2865 } 2866 2867 static int acpi_nfit_remove(struct acpi_device *adev) 2868 { 2869 /* see acpi_nfit_destruct */ 2870 return 0; 2871 } 2872 2873 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event) 2874 { 2875 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 2876 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 2877 union acpi_object *obj; 2878 acpi_status status; 2879 int ret; 2880 2881 dev_dbg(dev, "%s: event: %d\n", __func__, event); 2882 2883 if (event != NFIT_NOTIFY_UPDATE) 2884 return; 2885 2886 if (!dev->driver) { 2887 /* dev->driver may be null if we're being removed */ 2888 dev_dbg(dev, "%s: no driver found for dev\n", __func__); 2889 return; 2890 } 2891 2892 if (!acpi_desc) { 2893 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 2894 if (!acpi_desc) 2895 return; 2896 acpi_nfit_desc_init(acpi_desc, dev); 2897 } else { 2898 /* 2899 * Finish previous registration before considering new 2900 * regions. 2901 */ 2902 flush_workqueue(nfit_wq); 2903 } 2904 2905 /* Evaluate _FIT */ 2906 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf); 2907 if (ACPI_FAILURE(status)) { 2908 dev_err(dev, "failed to evaluate _FIT\n"); 2909 return; 2910 } 2911 2912 obj = buf.pointer; 2913 if (obj->type == ACPI_TYPE_BUFFER) { 2914 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 2915 obj->buffer.length); 2916 if (ret) 2917 dev_err(dev, "failed to merge updated NFIT\n"); 2918 } else 2919 dev_err(dev, "Invalid _FIT\n"); 2920 kfree(buf.pointer); 2921 } 2922 EXPORT_SYMBOL_GPL(__acpi_nfit_notify); 2923 2924 static void acpi_nfit_notify(struct acpi_device *adev, u32 event) 2925 { 2926 device_lock(&adev->dev); 2927 __acpi_nfit_notify(&adev->dev, adev->handle, event); 2928 device_unlock(&adev->dev); 2929 } 2930 2931 static const struct acpi_device_id acpi_nfit_ids[] = { 2932 { "ACPI0012", 0 }, 2933 { "", 0 }, 2934 }; 2935 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); 2936 2937 static struct acpi_driver acpi_nfit_driver = { 2938 .name = KBUILD_MODNAME, 2939 .ids = acpi_nfit_ids, 2940 .ops = { 2941 .add = acpi_nfit_add, 2942 .remove = acpi_nfit_remove, 2943 .notify = acpi_nfit_notify, 2944 }, 2945 }; 2946 2947 static __init int nfit_init(void) 2948 { 2949 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); 2950 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); 2951 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); 2952 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); 2953 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); 2954 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); 2955 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); 2956 2957 acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]); 2958 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]); 2959 acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]); 2960 acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]); 2961 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]); 2962 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]); 2963 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]); 2964 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]); 2965 acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]); 2966 acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]); 2967 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE1, nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); 2968 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE2, nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); 2969 acpi_str_to_uuid(UUID_NFIT_DIMM_N_MSFT, nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); 2970 2971 nfit_wq = create_singlethread_workqueue("nfit"); 2972 if (!nfit_wq) 2973 return -ENOMEM; 2974 2975 nfit_mce_register(); 2976 2977 return acpi_bus_register_driver(&acpi_nfit_driver); 2978 } 2979 2980 static __exit void nfit_exit(void) 2981 { 2982 nfit_mce_unregister(); 2983 acpi_bus_unregister_driver(&acpi_nfit_driver); 2984 destroy_workqueue(nfit_wq); 2985 WARN_ON(!list_empty(&acpi_descs)); 2986 } 2987 2988 module_init(nfit_init); 2989 module_exit(nfit_exit); 2990 MODULE_LICENSE("GPL v2"); 2991 MODULE_AUTHOR("Intel Corporation"); 2992