1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/list_sort.h> 14 #include <linux/libnvdimm.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/ndctl.h> 18 #include <linux/sysfs.h> 19 #include <linux/delay.h> 20 #include <linux/list.h> 21 #include <linux/acpi.h> 22 #include <linux/sort.h> 23 #include <linux/io.h> 24 #include <linux/nd.h> 25 #include <asm/cacheflush.h> 26 #include <acpi/nfit.h> 27 #include "nfit.h" 28 29 /* 30 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 31 * irrelevant. 32 */ 33 #include <linux/io-64-nonatomic-hi-lo.h> 34 35 static bool force_enable_dimms; 36 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); 37 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); 38 39 static bool disable_vendor_specific; 40 module_param(disable_vendor_specific, bool, S_IRUGO); 41 MODULE_PARM_DESC(disable_vendor_specific, 42 "Limit commands to the publicly specified set"); 43 44 static unsigned long override_dsm_mask; 45 module_param(override_dsm_mask, ulong, S_IRUGO); 46 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions"); 47 48 static int default_dsm_family = -1; 49 module_param(default_dsm_family, int, S_IRUGO); 50 MODULE_PARM_DESC(default_dsm_family, 51 "Try this DSM type first when identifying NVDIMM family"); 52 53 static bool no_init_ars; 54 module_param(no_init_ars, bool, 0644); 55 MODULE_PARM_DESC(no_init_ars, "Skip ARS run at nfit init time"); 56 57 LIST_HEAD(acpi_descs); 58 DEFINE_MUTEX(acpi_desc_lock); 59 60 static struct workqueue_struct *nfit_wq; 61 62 struct nfit_table_prev { 63 struct list_head spas; 64 struct list_head memdevs; 65 struct list_head dcrs; 66 struct list_head bdws; 67 struct list_head idts; 68 struct list_head flushes; 69 }; 70 71 static guid_t nfit_uuid[NFIT_UUID_MAX]; 72 73 const guid_t *to_nfit_uuid(enum nfit_uuids id) 74 { 75 return &nfit_uuid[id]; 76 } 77 EXPORT_SYMBOL(to_nfit_uuid); 78 79 static struct acpi_nfit_desc *to_acpi_nfit_desc( 80 struct nvdimm_bus_descriptor *nd_desc) 81 { 82 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); 83 } 84 85 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) 86 { 87 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 88 89 /* 90 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct 91 * acpi_device. 92 */ 93 if (!nd_desc->provider_name 94 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) 95 return NULL; 96 97 return to_acpi_device(acpi_desc->dev); 98 } 99 100 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status) 101 { 102 struct nd_cmd_clear_error *clear_err; 103 struct nd_cmd_ars_status *ars_status; 104 u16 flags; 105 106 switch (cmd) { 107 case ND_CMD_ARS_CAP: 108 if ((status & 0xffff) == NFIT_ARS_CAP_NONE) 109 return -ENOTTY; 110 111 /* Command failed */ 112 if (status & 0xffff) 113 return -EIO; 114 115 /* No supported scan types for this range */ 116 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; 117 if ((status >> 16 & flags) == 0) 118 return -ENOTTY; 119 return 0; 120 case ND_CMD_ARS_START: 121 /* ARS is in progress */ 122 if ((status & 0xffff) == NFIT_ARS_START_BUSY) 123 return -EBUSY; 124 125 /* Command failed */ 126 if (status & 0xffff) 127 return -EIO; 128 return 0; 129 case ND_CMD_ARS_STATUS: 130 ars_status = buf; 131 /* Command failed */ 132 if (status & 0xffff) 133 return -EIO; 134 /* Check extended status (Upper two bytes) */ 135 if (status == NFIT_ARS_STATUS_DONE) 136 return 0; 137 138 /* ARS is in progress */ 139 if (status == NFIT_ARS_STATUS_BUSY) 140 return -EBUSY; 141 142 /* No ARS performed for the current boot */ 143 if (status == NFIT_ARS_STATUS_NONE) 144 return -EAGAIN; 145 146 /* 147 * ARS interrupted, either we overflowed or some other 148 * agent wants the scan to stop. If we didn't overflow 149 * then just continue with the returned results. 150 */ 151 if (status == NFIT_ARS_STATUS_INTR) { 152 if (ars_status->out_length >= 40 && (ars_status->flags 153 & NFIT_ARS_F_OVERFLOW)) 154 return -ENOSPC; 155 return 0; 156 } 157 158 /* Unknown status */ 159 if (status >> 16) 160 return -EIO; 161 return 0; 162 case ND_CMD_CLEAR_ERROR: 163 clear_err = buf; 164 if (status & 0xffff) 165 return -EIO; 166 if (!clear_err->cleared) 167 return -EIO; 168 if (clear_err->length > clear_err->cleared) 169 return clear_err->cleared; 170 return 0; 171 default: 172 break; 173 } 174 175 /* all other non-zero status results in an error */ 176 if (status) 177 return -EIO; 178 return 0; 179 } 180 181 #define ACPI_LABELS_LOCKED 3 182 183 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 184 u32 status) 185 { 186 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 187 188 switch (cmd) { 189 case ND_CMD_GET_CONFIG_SIZE: 190 /* 191 * In the _LSI, _LSR, _LSW case the locked status is 192 * communicated via the read/write commands 193 */ 194 if (nfit_mem->has_lsr) 195 break; 196 197 if (status >> 16 & ND_CONFIG_LOCKED) 198 return -EACCES; 199 break; 200 case ND_CMD_GET_CONFIG_DATA: 201 if (nfit_mem->has_lsr && status == ACPI_LABELS_LOCKED) 202 return -EACCES; 203 break; 204 case ND_CMD_SET_CONFIG_DATA: 205 if (nfit_mem->has_lsw && status == ACPI_LABELS_LOCKED) 206 return -EACCES; 207 break; 208 default: 209 break; 210 } 211 212 /* all other non-zero status results in an error */ 213 if (status) 214 return -EIO; 215 return 0; 216 } 217 218 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 219 u32 status) 220 { 221 if (!nvdimm) 222 return xlat_bus_status(buf, cmd, status); 223 return xlat_nvdimm_status(nvdimm, buf, cmd, status); 224 } 225 226 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */ 227 static union acpi_object *pkg_to_buf(union acpi_object *pkg) 228 { 229 int i; 230 void *dst; 231 size_t size = 0; 232 union acpi_object *buf = NULL; 233 234 if (pkg->type != ACPI_TYPE_PACKAGE) { 235 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 236 pkg->type); 237 goto err; 238 } 239 240 for (i = 0; i < pkg->package.count; i++) { 241 union acpi_object *obj = &pkg->package.elements[i]; 242 243 if (obj->type == ACPI_TYPE_INTEGER) 244 size += 4; 245 else if (obj->type == ACPI_TYPE_BUFFER) 246 size += obj->buffer.length; 247 else { 248 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 249 obj->type); 250 goto err; 251 } 252 } 253 254 buf = ACPI_ALLOCATE(sizeof(*buf) + size); 255 if (!buf) 256 goto err; 257 258 dst = buf + 1; 259 buf->type = ACPI_TYPE_BUFFER; 260 buf->buffer.length = size; 261 buf->buffer.pointer = dst; 262 for (i = 0; i < pkg->package.count; i++) { 263 union acpi_object *obj = &pkg->package.elements[i]; 264 265 if (obj->type == ACPI_TYPE_INTEGER) { 266 memcpy(dst, &obj->integer.value, 4); 267 dst += 4; 268 } else if (obj->type == ACPI_TYPE_BUFFER) { 269 memcpy(dst, obj->buffer.pointer, obj->buffer.length); 270 dst += obj->buffer.length; 271 } 272 } 273 err: 274 ACPI_FREE(pkg); 275 return buf; 276 } 277 278 static union acpi_object *int_to_buf(union acpi_object *integer) 279 { 280 union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4); 281 void *dst = NULL; 282 283 if (!buf) 284 goto err; 285 286 if (integer->type != ACPI_TYPE_INTEGER) { 287 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 288 integer->type); 289 goto err; 290 } 291 292 dst = buf + 1; 293 buf->type = ACPI_TYPE_BUFFER; 294 buf->buffer.length = 4; 295 buf->buffer.pointer = dst; 296 memcpy(dst, &integer->integer.value, 4); 297 err: 298 ACPI_FREE(integer); 299 return buf; 300 } 301 302 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset, 303 u32 len, void *data) 304 { 305 acpi_status rc; 306 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 307 struct acpi_object_list input = { 308 .count = 3, 309 .pointer = (union acpi_object []) { 310 [0] = { 311 .integer.type = ACPI_TYPE_INTEGER, 312 .integer.value = offset, 313 }, 314 [1] = { 315 .integer.type = ACPI_TYPE_INTEGER, 316 .integer.value = len, 317 }, 318 [2] = { 319 .buffer.type = ACPI_TYPE_BUFFER, 320 .buffer.pointer = data, 321 .buffer.length = len, 322 }, 323 }, 324 }; 325 326 rc = acpi_evaluate_object(handle, "_LSW", &input, &buf); 327 if (ACPI_FAILURE(rc)) 328 return NULL; 329 return int_to_buf(buf.pointer); 330 } 331 332 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset, 333 u32 len) 334 { 335 acpi_status rc; 336 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 337 struct acpi_object_list input = { 338 .count = 2, 339 .pointer = (union acpi_object []) { 340 [0] = { 341 .integer.type = ACPI_TYPE_INTEGER, 342 .integer.value = offset, 343 }, 344 [1] = { 345 .integer.type = ACPI_TYPE_INTEGER, 346 .integer.value = len, 347 }, 348 }, 349 }; 350 351 rc = acpi_evaluate_object(handle, "_LSR", &input, &buf); 352 if (ACPI_FAILURE(rc)) 353 return NULL; 354 return pkg_to_buf(buf.pointer); 355 } 356 357 static union acpi_object *acpi_label_info(acpi_handle handle) 358 { 359 acpi_status rc; 360 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 361 362 rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf); 363 if (ACPI_FAILURE(rc)) 364 return NULL; 365 return pkg_to_buf(buf.pointer); 366 } 367 368 static u8 nfit_dsm_revid(unsigned family, unsigned func) 369 { 370 static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = { 371 [NVDIMM_FAMILY_INTEL] = { 372 [NVDIMM_INTEL_GET_MODES] = 2, 373 [NVDIMM_INTEL_GET_FWINFO] = 2, 374 [NVDIMM_INTEL_START_FWUPDATE] = 2, 375 [NVDIMM_INTEL_SEND_FWUPDATE] = 2, 376 [NVDIMM_INTEL_FINISH_FWUPDATE] = 2, 377 [NVDIMM_INTEL_QUERY_FWUPDATE] = 2, 378 [NVDIMM_INTEL_SET_THRESHOLD] = 2, 379 [NVDIMM_INTEL_INJECT_ERROR] = 2, 380 }, 381 }; 382 u8 id; 383 384 if (family > NVDIMM_FAMILY_MAX) 385 return 0; 386 if (func > 31) 387 return 0; 388 id = revid_table[family][func]; 389 if (id == 0) 390 return 1; /* default */ 391 return id; 392 } 393 394 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, 395 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) 396 { 397 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 398 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 399 union acpi_object in_obj, in_buf, *out_obj; 400 const struct nd_cmd_desc *desc = NULL; 401 struct device *dev = acpi_desc->dev; 402 struct nd_cmd_pkg *call_pkg = NULL; 403 const char *cmd_name, *dimm_name; 404 unsigned long cmd_mask, dsm_mask; 405 u32 offset, fw_status = 0; 406 acpi_handle handle; 407 unsigned int func; 408 const guid_t *guid; 409 int rc, i; 410 411 if (cmd_rc) 412 *cmd_rc = -EINVAL; 413 func = cmd; 414 if (cmd == ND_CMD_CALL) { 415 call_pkg = buf; 416 func = call_pkg->nd_command; 417 418 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++) 419 if (call_pkg->nd_reserved2[i]) 420 return -EINVAL; 421 } 422 423 if (nvdimm) { 424 struct acpi_device *adev = nfit_mem->adev; 425 426 if (!adev) 427 return -ENOTTY; 428 if (call_pkg && nfit_mem->family != call_pkg->nd_family) 429 return -ENOTTY; 430 431 dimm_name = nvdimm_name(nvdimm); 432 cmd_name = nvdimm_cmd_name(cmd); 433 cmd_mask = nvdimm_cmd_mask(nvdimm); 434 dsm_mask = nfit_mem->dsm_mask; 435 desc = nd_cmd_dimm_desc(cmd); 436 guid = to_nfit_uuid(nfit_mem->family); 437 handle = adev->handle; 438 } else { 439 struct acpi_device *adev = to_acpi_dev(acpi_desc); 440 441 cmd_name = nvdimm_bus_cmd_name(cmd); 442 cmd_mask = nd_desc->cmd_mask; 443 dsm_mask = cmd_mask; 444 if (cmd == ND_CMD_CALL) 445 dsm_mask = nd_desc->bus_dsm_mask; 446 desc = nd_cmd_bus_desc(cmd); 447 guid = to_nfit_uuid(NFIT_DEV_BUS); 448 handle = adev->handle; 449 dimm_name = "bus"; 450 } 451 452 if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) 453 return -ENOTTY; 454 455 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) 456 return -ENOTTY; 457 458 in_obj.type = ACPI_TYPE_PACKAGE; 459 in_obj.package.count = 1; 460 in_obj.package.elements = &in_buf; 461 in_buf.type = ACPI_TYPE_BUFFER; 462 in_buf.buffer.pointer = buf; 463 in_buf.buffer.length = 0; 464 465 /* libnvdimm has already validated the input envelope */ 466 for (i = 0; i < desc->in_num; i++) 467 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, 468 i, buf); 469 470 if (call_pkg) { 471 /* skip over package wrapper */ 472 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; 473 in_buf.buffer.length = call_pkg->nd_size_in; 474 } 475 476 dev_dbg(dev, "%s cmd: %d: func: %d input length: %d\n", 477 dimm_name, cmd, func, in_buf.buffer.length); 478 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, 479 in_buf.buffer.pointer, 480 min_t(u32, 256, in_buf.buffer.length), true); 481 482 /* call the BIOS, prefer the named methods over _DSM if available */ 483 if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE && nfit_mem->has_lsr) 484 out_obj = acpi_label_info(handle); 485 else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && nfit_mem->has_lsr) { 486 struct nd_cmd_get_config_data_hdr *p = buf; 487 488 out_obj = acpi_label_read(handle, p->in_offset, p->in_length); 489 } else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA 490 && nfit_mem->has_lsw) { 491 struct nd_cmd_set_config_hdr *p = buf; 492 493 out_obj = acpi_label_write(handle, p->in_offset, p->in_length, 494 p->in_buf); 495 } else { 496 u8 revid; 497 498 if (nvdimm) 499 revid = nfit_dsm_revid(nfit_mem->family, func); 500 else 501 revid = 1; 502 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj); 503 } 504 505 if (!out_obj) { 506 dev_dbg(dev, "%s _DSM failed cmd: %s\n", dimm_name, cmd_name); 507 return -EINVAL; 508 } 509 510 if (call_pkg) { 511 call_pkg->nd_fw_size = out_obj->buffer.length; 512 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, 513 out_obj->buffer.pointer, 514 min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); 515 516 ACPI_FREE(out_obj); 517 /* 518 * Need to support FW function w/o known size in advance. 519 * Caller can determine required size based upon nd_fw_size. 520 * If we return an error (like elsewhere) then caller wouldn't 521 * be able to rely upon data returned to make calculation. 522 */ 523 if (cmd_rc) 524 *cmd_rc = 0; 525 return 0; 526 } 527 528 if (out_obj->package.type != ACPI_TYPE_BUFFER) { 529 dev_dbg(dev, "%s unexpected output object type cmd: %s type: %d\n", 530 dimm_name, cmd_name, out_obj->type); 531 rc = -EINVAL; 532 goto out; 533 } 534 535 dev_dbg(dev, "%s cmd: %s output length: %d\n", dimm_name, 536 cmd_name, out_obj->buffer.length); 537 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4, 538 out_obj->buffer.pointer, 539 min_t(u32, 128, out_obj->buffer.length), true); 540 541 for (i = 0, offset = 0; i < desc->out_num; i++) { 542 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, 543 (u32 *) out_obj->buffer.pointer, 544 out_obj->buffer.length - offset); 545 546 if (offset + out_size > out_obj->buffer.length) { 547 dev_dbg(dev, "%s output object underflow cmd: %s field: %d\n", 548 dimm_name, cmd_name, i); 549 break; 550 } 551 552 if (in_buf.buffer.length + offset + out_size > buf_len) { 553 dev_dbg(dev, "%s output overrun cmd: %s field: %d\n", 554 dimm_name, cmd_name, i); 555 rc = -ENXIO; 556 goto out; 557 } 558 memcpy(buf + in_buf.buffer.length + offset, 559 out_obj->buffer.pointer + offset, out_size); 560 offset += out_size; 561 } 562 563 /* 564 * Set fw_status for all the commands with a known format to be 565 * later interpreted by xlat_status(). 566 */ 567 if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP 568 && cmd <= ND_CMD_CLEAR_ERROR) 569 || (nvdimm && cmd >= ND_CMD_SMART 570 && cmd <= ND_CMD_VENDOR))) 571 fw_status = *(u32 *) out_obj->buffer.pointer; 572 573 if (offset + in_buf.buffer.length < buf_len) { 574 if (i >= 1) { 575 /* 576 * status valid, return the number of bytes left 577 * unfilled in the output buffer 578 */ 579 rc = buf_len - offset - in_buf.buffer.length; 580 if (cmd_rc) 581 *cmd_rc = xlat_status(nvdimm, buf, cmd, 582 fw_status); 583 } else { 584 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", 585 __func__, dimm_name, cmd_name, buf_len, 586 offset); 587 rc = -ENXIO; 588 } 589 } else { 590 rc = 0; 591 if (cmd_rc) 592 *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status); 593 } 594 595 out: 596 ACPI_FREE(out_obj); 597 598 return rc; 599 } 600 EXPORT_SYMBOL_GPL(acpi_nfit_ctl); 601 602 static const char *spa_type_name(u16 type) 603 { 604 static const char *to_name[] = { 605 [NFIT_SPA_VOLATILE] = "volatile", 606 [NFIT_SPA_PM] = "pmem", 607 [NFIT_SPA_DCR] = "dimm-control-region", 608 [NFIT_SPA_BDW] = "block-data-window", 609 [NFIT_SPA_VDISK] = "volatile-disk", 610 [NFIT_SPA_VCD] = "volatile-cd", 611 [NFIT_SPA_PDISK] = "persistent-disk", 612 [NFIT_SPA_PCD] = "persistent-cd", 613 614 }; 615 616 if (type > NFIT_SPA_PCD) 617 return "unknown"; 618 619 return to_name[type]; 620 } 621 622 int nfit_spa_type(struct acpi_nfit_system_address *spa) 623 { 624 int i; 625 626 for (i = 0; i < NFIT_UUID_MAX; i++) 627 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid)) 628 return i; 629 return -1; 630 } 631 632 static bool add_spa(struct acpi_nfit_desc *acpi_desc, 633 struct nfit_table_prev *prev, 634 struct acpi_nfit_system_address *spa) 635 { 636 struct device *dev = acpi_desc->dev; 637 struct nfit_spa *nfit_spa; 638 639 if (spa->header.length != sizeof(*spa)) 640 return false; 641 642 list_for_each_entry(nfit_spa, &prev->spas, list) { 643 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { 644 list_move_tail(&nfit_spa->list, &acpi_desc->spas); 645 return true; 646 } 647 } 648 649 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), 650 GFP_KERNEL); 651 if (!nfit_spa) 652 return false; 653 INIT_LIST_HEAD(&nfit_spa->list); 654 memcpy(nfit_spa->spa, spa, sizeof(*spa)); 655 list_add_tail(&nfit_spa->list, &acpi_desc->spas); 656 dev_dbg(dev, "spa index: %d type: %s\n", 657 spa->range_index, 658 spa_type_name(nfit_spa_type(spa))); 659 return true; 660 } 661 662 static bool add_memdev(struct acpi_nfit_desc *acpi_desc, 663 struct nfit_table_prev *prev, 664 struct acpi_nfit_memory_map *memdev) 665 { 666 struct device *dev = acpi_desc->dev; 667 struct nfit_memdev *nfit_memdev; 668 669 if (memdev->header.length != sizeof(*memdev)) 670 return false; 671 672 list_for_each_entry(nfit_memdev, &prev->memdevs, list) 673 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { 674 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); 675 return true; 676 } 677 678 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), 679 GFP_KERNEL); 680 if (!nfit_memdev) 681 return false; 682 INIT_LIST_HEAD(&nfit_memdev->list); 683 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); 684 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); 685 dev_dbg(dev, "memdev handle: %#x spa: %d dcr: %d flags: %#x\n", 686 memdev->device_handle, memdev->range_index, 687 memdev->region_index, memdev->flags); 688 return true; 689 } 690 691 int nfit_get_smbios_id(u32 device_handle, u16 *flags) 692 { 693 struct acpi_nfit_memory_map *memdev; 694 struct acpi_nfit_desc *acpi_desc; 695 struct nfit_mem *nfit_mem; 696 697 mutex_lock(&acpi_desc_lock); 698 list_for_each_entry(acpi_desc, &acpi_descs, list) { 699 mutex_lock(&acpi_desc->init_mutex); 700 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 701 memdev = __to_nfit_memdev(nfit_mem); 702 if (memdev->device_handle == device_handle) { 703 mutex_unlock(&acpi_desc->init_mutex); 704 mutex_unlock(&acpi_desc_lock); 705 *flags = memdev->flags; 706 return memdev->physical_id; 707 } 708 } 709 mutex_unlock(&acpi_desc->init_mutex); 710 } 711 mutex_unlock(&acpi_desc_lock); 712 713 return -ENODEV; 714 } 715 EXPORT_SYMBOL_GPL(nfit_get_smbios_id); 716 717 /* 718 * An implementation may provide a truncated control region if no block windows 719 * are defined. 720 */ 721 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) 722 { 723 if (dcr->header.length < offsetof(struct acpi_nfit_control_region, 724 window_size)) 725 return 0; 726 if (dcr->windows) 727 return sizeof(*dcr); 728 return offsetof(struct acpi_nfit_control_region, window_size); 729 } 730 731 static bool add_dcr(struct acpi_nfit_desc *acpi_desc, 732 struct nfit_table_prev *prev, 733 struct acpi_nfit_control_region *dcr) 734 { 735 struct device *dev = acpi_desc->dev; 736 struct nfit_dcr *nfit_dcr; 737 738 if (!sizeof_dcr(dcr)) 739 return false; 740 741 list_for_each_entry(nfit_dcr, &prev->dcrs, list) 742 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { 743 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); 744 return true; 745 } 746 747 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), 748 GFP_KERNEL); 749 if (!nfit_dcr) 750 return false; 751 INIT_LIST_HEAD(&nfit_dcr->list); 752 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); 753 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); 754 dev_dbg(dev, "dcr index: %d windows: %d\n", 755 dcr->region_index, dcr->windows); 756 return true; 757 } 758 759 static bool add_bdw(struct acpi_nfit_desc *acpi_desc, 760 struct nfit_table_prev *prev, 761 struct acpi_nfit_data_region *bdw) 762 { 763 struct device *dev = acpi_desc->dev; 764 struct nfit_bdw *nfit_bdw; 765 766 if (bdw->header.length != sizeof(*bdw)) 767 return false; 768 list_for_each_entry(nfit_bdw, &prev->bdws, list) 769 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { 770 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); 771 return true; 772 } 773 774 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), 775 GFP_KERNEL); 776 if (!nfit_bdw) 777 return false; 778 INIT_LIST_HEAD(&nfit_bdw->list); 779 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); 780 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); 781 dev_dbg(dev, "bdw dcr: %d windows: %d\n", 782 bdw->region_index, bdw->windows); 783 return true; 784 } 785 786 static size_t sizeof_idt(struct acpi_nfit_interleave *idt) 787 { 788 if (idt->header.length < sizeof(*idt)) 789 return 0; 790 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); 791 } 792 793 static bool add_idt(struct acpi_nfit_desc *acpi_desc, 794 struct nfit_table_prev *prev, 795 struct acpi_nfit_interleave *idt) 796 { 797 struct device *dev = acpi_desc->dev; 798 struct nfit_idt *nfit_idt; 799 800 if (!sizeof_idt(idt)) 801 return false; 802 803 list_for_each_entry(nfit_idt, &prev->idts, list) { 804 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) 805 continue; 806 807 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { 808 list_move_tail(&nfit_idt->list, &acpi_desc->idts); 809 return true; 810 } 811 } 812 813 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), 814 GFP_KERNEL); 815 if (!nfit_idt) 816 return false; 817 INIT_LIST_HEAD(&nfit_idt->list); 818 memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); 819 list_add_tail(&nfit_idt->list, &acpi_desc->idts); 820 dev_dbg(dev, "idt index: %d num_lines: %d\n", 821 idt->interleave_index, idt->line_count); 822 return true; 823 } 824 825 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) 826 { 827 if (flush->header.length < sizeof(*flush)) 828 return 0; 829 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); 830 } 831 832 static bool add_flush(struct acpi_nfit_desc *acpi_desc, 833 struct nfit_table_prev *prev, 834 struct acpi_nfit_flush_address *flush) 835 { 836 struct device *dev = acpi_desc->dev; 837 struct nfit_flush *nfit_flush; 838 839 if (!sizeof_flush(flush)) 840 return false; 841 842 list_for_each_entry(nfit_flush, &prev->flushes, list) { 843 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) 844 continue; 845 846 if (memcmp(nfit_flush->flush, flush, 847 sizeof_flush(flush)) == 0) { 848 list_move_tail(&nfit_flush->list, &acpi_desc->flushes); 849 return true; 850 } 851 } 852 853 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) 854 + sizeof_flush(flush), GFP_KERNEL); 855 if (!nfit_flush) 856 return false; 857 INIT_LIST_HEAD(&nfit_flush->list); 858 memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); 859 list_add_tail(&nfit_flush->list, &acpi_desc->flushes); 860 dev_dbg(dev, "nfit_flush handle: %d hint_count: %d\n", 861 flush->device_handle, flush->hint_count); 862 return true; 863 } 864 865 static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc, 866 struct acpi_nfit_capabilities *pcap) 867 { 868 struct device *dev = acpi_desc->dev; 869 u32 mask; 870 871 mask = (1 << (pcap->highest_capability + 1)) - 1; 872 acpi_desc->platform_cap = pcap->capabilities & mask; 873 dev_dbg(dev, "cap: %#x\n", acpi_desc->platform_cap); 874 return true; 875 } 876 877 static void *add_table(struct acpi_nfit_desc *acpi_desc, 878 struct nfit_table_prev *prev, void *table, const void *end) 879 { 880 struct device *dev = acpi_desc->dev; 881 struct acpi_nfit_header *hdr; 882 void *err = ERR_PTR(-ENOMEM); 883 884 if (table >= end) 885 return NULL; 886 887 hdr = table; 888 if (!hdr->length) { 889 dev_warn(dev, "found a zero length table '%d' parsing nfit\n", 890 hdr->type); 891 return NULL; 892 } 893 894 switch (hdr->type) { 895 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: 896 if (!add_spa(acpi_desc, prev, table)) 897 return err; 898 break; 899 case ACPI_NFIT_TYPE_MEMORY_MAP: 900 if (!add_memdev(acpi_desc, prev, table)) 901 return err; 902 break; 903 case ACPI_NFIT_TYPE_CONTROL_REGION: 904 if (!add_dcr(acpi_desc, prev, table)) 905 return err; 906 break; 907 case ACPI_NFIT_TYPE_DATA_REGION: 908 if (!add_bdw(acpi_desc, prev, table)) 909 return err; 910 break; 911 case ACPI_NFIT_TYPE_INTERLEAVE: 912 if (!add_idt(acpi_desc, prev, table)) 913 return err; 914 break; 915 case ACPI_NFIT_TYPE_FLUSH_ADDRESS: 916 if (!add_flush(acpi_desc, prev, table)) 917 return err; 918 break; 919 case ACPI_NFIT_TYPE_SMBIOS: 920 dev_dbg(dev, "smbios\n"); 921 break; 922 case ACPI_NFIT_TYPE_CAPABILITIES: 923 if (!add_platform_cap(acpi_desc, table)) 924 return err; 925 break; 926 default: 927 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); 928 break; 929 } 930 931 return table + hdr->length; 932 } 933 934 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, 935 struct nfit_mem *nfit_mem) 936 { 937 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 938 u16 dcr = nfit_mem->dcr->region_index; 939 struct nfit_spa *nfit_spa; 940 941 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 942 u16 range_index = nfit_spa->spa->range_index; 943 int type = nfit_spa_type(nfit_spa->spa); 944 struct nfit_memdev *nfit_memdev; 945 946 if (type != NFIT_SPA_BDW) 947 continue; 948 949 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 950 if (nfit_memdev->memdev->range_index != range_index) 951 continue; 952 if (nfit_memdev->memdev->device_handle != device_handle) 953 continue; 954 if (nfit_memdev->memdev->region_index != dcr) 955 continue; 956 957 nfit_mem->spa_bdw = nfit_spa->spa; 958 return; 959 } 960 } 961 962 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", 963 nfit_mem->spa_dcr->range_index); 964 nfit_mem->bdw = NULL; 965 } 966 967 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, 968 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) 969 { 970 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; 971 struct nfit_memdev *nfit_memdev; 972 struct nfit_bdw *nfit_bdw; 973 struct nfit_idt *nfit_idt; 974 u16 idt_idx, range_index; 975 976 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { 977 if (nfit_bdw->bdw->region_index != dcr) 978 continue; 979 nfit_mem->bdw = nfit_bdw->bdw; 980 break; 981 } 982 983 if (!nfit_mem->bdw) 984 return; 985 986 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); 987 988 if (!nfit_mem->spa_bdw) 989 return; 990 991 range_index = nfit_mem->spa_bdw->range_index; 992 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 993 if (nfit_memdev->memdev->range_index != range_index || 994 nfit_memdev->memdev->region_index != dcr) 995 continue; 996 nfit_mem->memdev_bdw = nfit_memdev->memdev; 997 idt_idx = nfit_memdev->memdev->interleave_index; 998 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 999 if (nfit_idt->idt->interleave_index != idt_idx) 1000 continue; 1001 nfit_mem->idt_bdw = nfit_idt->idt; 1002 break; 1003 } 1004 break; 1005 } 1006 } 1007 1008 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc, 1009 struct acpi_nfit_system_address *spa) 1010 { 1011 struct nfit_mem *nfit_mem, *found; 1012 struct nfit_memdev *nfit_memdev; 1013 int type = spa ? nfit_spa_type(spa) : 0; 1014 1015 switch (type) { 1016 case NFIT_SPA_DCR: 1017 case NFIT_SPA_PM: 1018 break; 1019 default: 1020 if (spa) 1021 return 0; 1022 } 1023 1024 /* 1025 * This loop runs in two modes, when a dimm is mapped the loop 1026 * adds memdev associations to an existing dimm, or creates a 1027 * dimm. In the unmapped dimm case this loop sweeps for memdev 1028 * instances with an invalid / zero range_index and adds those 1029 * dimms without spa associations. 1030 */ 1031 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1032 struct nfit_flush *nfit_flush; 1033 struct nfit_dcr *nfit_dcr; 1034 u32 device_handle; 1035 u16 dcr; 1036 1037 if (spa && nfit_memdev->memdev->range_index != spa->range_index) 1038 continue; 1039 if (!spa && nfit_memdev->memdev->range_index) 1040 continue; 1041 found = NULL; 1042 dcr = nfit_memdev->memdev->region_index; 1043 device_handle = nfit_memdev->memdev->device_handle; 1044 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1045 if (__to_nfit_memdev(nfit_mem)->device_handle 1046 == device_handle) { 1047 found = nfit_mem; 1048 break; 1049 } 1050 1051 if (found) 1052 nfit_mem = found; 1053 else { 1054 nfit_mem = devm_kzalloc(acpi_desc->dev, 1055 sizeof(*nfit_mem), GFP_KERNEL); 1056 if (!nfit_mem) 1057 return -ENOMEM; 1058 INIT_LIST_HEAD(&nfit_mem->list); 1059 nfit_mem->acpi_desc = acpi_desc; 1060 list_add(&nfit_mem->list, &acpi_desc->dimms); 1061 } 1062 1063 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1064 if (nfit_dcr->dcr->region_index != dcr) 1065 continue; 1066 /* 1067 * Record the control region for the dimm. For 1068 * the ACPI 6.1 case, where there are separate 1069 * control regions for the pmem vs blk 1070 * interfaces, be sure to record the extended 1071 * blk details. 1072 */ 1073 if (!nfit_mem->dcr) 1074 nfit_mem->dcr = nfit_dcr->dcr; 1075 else if (nfit_mem->dcr->windows == 0 1076 && nfit_dcr->dcr->windows) 1077 nfit_mem->dcr = nfit_dcr->dcr; 1078 break; 1079 } 1080 1081 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { 1082 struct acpi_nfit_flush_address *flush; 1083 u16 i; 1084 1085 if (nfit_flush->flush->device_handle != device_handle) 1086 continue; 1087 nfit_mem->nfit_flush = nfit_flush; 1088 flush = nfit_flush->flush; 1089 nfit_mem->flush_wpq = devm_kcalloc(acpi_desc->dev, 1090 flush->hint_count, 1091 sizeof(struct resource), 1092 GFP_KERNEL); 1093 if (!nfit_mem->flush_wpq) 1094 return -ENOMEM; 1095 for (i = 0; i < flush->hint_count; i++) { 1096 struct resource *res = &nfit_mem->flush_wpq[i]; 1097 1098 res->start = flush->hint_address[i]; 1099 res->end = res->start + 8 - 1; 1100 } 1101 break; 1102 } 1103 1104 if (dcr && !nfit_mem->dcr) { 1105 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", 1106 spa->range_index, dcr); 1107 return -ENODEV; 1108 } 1109 1110 if (type == NFIT_SPA_DCR) { 1111 struct nfit_idt *nfit_idt; 1112 u16 idt_idx; 1113 1114 /* multiple dimms may share a SPA when interleaved */ 1115 nfit_mem->spa_dcr = spa; 1116 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1117 idt_idx = nfit_memdev->memdev->interleave_index; 1118 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 1119 if (nfit_idt->idt->interleave_index != idt_idx) 1120 continue; 1121 nfit_mem->idt_dcr = nfit_idt->idt; 1122 break; 1123 } 1124 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); 1125 } else if (type == NFIT_SPA_PM) { 1126 /* 1127 * A single dimm may belong to multiple SPA-PM 1128 * ranges, record at least one in addition to 1129 * any SPA-DCR range. 1130 */ 1131 nfit_mem->memdev_pmem = nfit_memdev->memdev; 1132 } else 1133 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1134 } 1135 1136 return 0; 1137 } 1138 1139 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) 1140 { 1141 struct nfit_mem *a = container_of(_a, typeof(*a), list); 1142 struct nfit_mem *b = container_of(_b, typeof(*b), list); 1143 u32 handleA, handleB; 1144 1145 handleA = __to_nfit_memdev(a)->device_handle; 1146 handleB = __to_nfit_memdev(b)->device_handle; 1147 if (handleA < handleB) 1148 return -1; 1149 else if (handleA > handleB) 1150 return 1; 1151 return 0; 1152 } 1153 1154 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) 1155 { 1156 struct nfit_spa *nfit_spa; 1157 int rc; 1158 1159 1160 /* 1161 * For each SPA-DCR or SPA-PMEM address range find its 1162 * corresponding MEMDEV(s). From each MEMDEV find the 1163 * corresponding DCR. Then, if we're operating on a SPA-DCR, 1164 * try to find a SPA-BDW and a corresponding BDW that references 1165 * the DCR. Throw it all into an nfit_mem object. Note, that 1166 * BDWs are optional. 1167 */ 1168 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 1169 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa); 1170 if (rc) 1171 return rc; 1172 } 1173 1174 /* 1175 * If a DIMM has failed to be mapped into SPA there will be no 1176 * SPA entries above. Find and register all the unmapped DIMMs 1177 * for reporting and recovery purposes. 1178 */ 1179 rc = __nfit_mem_init(acpi_desc, NULL); 1180 if (rc) 1181 return rc; 1182 1183 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); 1184 1185 return 0; 1186 } 1187 1188 static ssize_t bus_dsm_mask_show(struct device *dev, 1189 struct device_attribute *attr, char *buf) 1190 { 1191 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1192 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1193 1194 return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask); 1195 } 1196 static struct device_attribute dev_attr_bus_dsm_mask = 1197 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL); 1198 1199 static ssize_t revision_show(struct device *dev, 1200 struct device_attribute *attr, char *buf) 1201 { 1202 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1203 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1204 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1205 1206 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); 1207 } 1208 static DEVICE_ATTR_RO(revision); 1209 1210 static ssize_t hw_error_scrub_show(struct device *dev, 1211 struct device_attribute *attr, char *buf) 1212 { 1213 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1214 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1215 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1216 1217 return sprintf(buf, "%d\n", acpi_desc->scrub_mode); 1218 } 1219 1220 /* 1221 * The 'hw_error_scrub' attribute can have the following values written to it: 1222 * '0': Switch to the default mode where an exception will only insert 1223 * the address of the memory error into the poison and badblocks lists. 1224 * '1': Enable a full scrub to happen if an exception for a memory error is 1225 * received. 1226 */ 1227 static ssize_t hw_error_scrub_store(struct device *dev, 1228 struct device_attribute *attr, const char *buf, size_t size) 1229 { 1230 struct nvdimm_bus_descriptor *nd_desc; 1231 ssize_t rc; 1232 long val; 1233 1234 rc = kstrtol(buf, 0, &val); 1235 if (rc) 1236 return rc; 1237 1238 device_lock(dev); 1239 nd_desc = dev_get_drvdata(dev); 1240 if (nd_desc) { 1241 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1242 1243 switch (val) { 1244 case HW_ERROR_SCRUB_ON: 1245 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON; 1246 break; 1247 case HW_ERROR_SCRUB_OFF: 1248 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF; 1249 break; 1250 default: 1251 rc = -EINVAL; 1252 break; 1253 } 1254 } 1255 device_unlock(dev); 1256 if (rc) 1257 return rc; 1258 return size; 1259 } 1260 static DEVICE_ATTR_RW(hw_error_scrub); 1261 1262 /* 1263 * This shows the number of full Address Range Scrubs that have been 1264 * completed since driver load time. Userspace can wait on this using 1265 * select/poll etc. A '+' at the end indicates an ARS is in progress 1266 */ 1267 static ssize_t scrub_show(struct device *dev, 1268 struct device_attribute *attr, char *buf) 1269 { 1270 struct nvdimm_bus_descriptor *nd_desc; 1271 ssize_t rc = -ENXIO; 1272 1273 device_lock(dev); 1274 nd_desc = dev_get_drvdata(dev); 1275 if (nd_desc) { 1276 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1277 1278 mutex_lock(&acpi_desc->init_mutex); 1279 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, 1280 acpi_desc->scrub_busy 1281 && !acpi_desc->cancel ? "+\n" : "\n"); 1282 mutex_unlock(&acpi_desc->init_mutex); 1283 } 1284 device_unlock(dev); 1285 return rc; 1286 } 1287 1288 static ssize_t scrub_store(struct device *dev, 1289 struct device_attribute *attr, const char *buf, size_t size) 1290 { 1291 struct nvdimm_bus_descriptor *nd_desc; 1292 ssize_t rc; 1293 long val; 1294 1295 rc = kstrtol(buf, 0, &val); 1296 if (rc) 1297 return rc; 1298 if (val != 1) 1299 return -EINVAL; 1300 1301 device_lock(dev); 1302 nd_desc = dev_get_drvdata(dev); 1303 if (nd_desc) { 1304 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1305 1306 rc = acpi_nfit_ars_rescan(acpi_desc, 0); 1307 } 1308 device_unlock(dev); 1309 if (rc) 1310 return rc; 1311 return size; 1312 } 1313 static DEVICE_ATTR_RW(scrub); 1314 1315 static bool ars_supported(struct nvdimm_bus *nvdimm_bus) 1316 { 1317 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1318 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START 1319 | 1 << ND_CMD_ARS_STATUS; 1320 1321 return (nd_desc->cmd_mask & mask) == mask; 1322 } 1323 1324 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) 1325 { 1326 struct device *dev = container_of(kobj, struct device, kobj); 1327 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1328 1329 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) 1330 return 0; 1331 return a->mode; 1332 } 1333 1334 static struct attribute *acpi_nfit_attributes[] = { 1335 &dev_attr_revision.attr, 1336 &dev_attr_scrub.attr, 1337 &dev_attr_hw_error_scrub.attr, 1338 &dev_attr_bus_dsm_mask.attr, 1339 NULL, 1340 }; 1341 1342 static const struct attribute_group acpi_nfit_attribute_group = { 1343 .name = "nfit", 1344 .attrs = acpi_nfit_attributes, 1345 .is_visible = nfit_visible, 1346 }; 1347 1348 static const struct attribute_group *acpi_nfit_attribute_groups[] = { 1349 &nvdimm_bus_attribute_group, 1350 &acpi_nfit_attribute_group, 1351 NULL, 1352 }; 1353 1354 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) 1355 { 1356 struct nvdimm *nvdimm = to_nvdimm(dev); 1357 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1358 1359 return __to_nfit_memdev(nfit_mem); 1360 } 1361 1362 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) 1363 { 1364 struct nvdimm *nvdimm = to_nvdimm(dev); 1365 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1366 1367 return nfit_mem->dcr; 1368 } 1369 1370 static ssize_t handle_show(struct device *dev, 1371 struct device_attribute *attr, char *buf) 1372 { 1373 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1374 1375 return sprintf(buf, "%#x\n", memdev->device_handle); 1376 } 1377 static DEVICE_ATTR_RO(handle); 1378 1379 static ssize_t phys_id_show(struct device *dev, 1380 struct device_attribute *attr, char *buf) 1381 { 1382 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1383 1384 return sprintf(buf, "%#x\n", memdev->physical_id); 1385 } 1386 static DEVICE_ATTR_RO(phys_id); 1387 1388 static ssize_t vendor_show(struct device *dev, 1389 struct device_attribute *attr, char *buf) 1390 { 1391 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1392 1393 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); 1394 } 1395 static DEVICE_ATTR_RO(vendor); 1396 1397 static ssize_t rev_id_show(struct device *dev, 1398 struct device_attribute *attr, char *buf) 1399 { 1400 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1401 1402 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); 1403 } 1404 static DEVICE_ATTR_RO(rev_id); 1405 1406 static ssize_t device_show(struct device *dev, 1407 struct device_attribute *attr, char *buf) 1408 { 1409 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1410 1411 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); 1412 } 1413 static DEVICE_ATTR_RO(device); 1414 1415 static ssize_t subsystem_vendor_show(struct device *dev, 1416 struct device_attribute *attr, char *buf) 1417 { 1418 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1419 1420 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); 1421 } 1422 static DEVICE_ATTR_RO(subsystem_vendor); 1423 1424 static ssize_t subsystem_rev_id_show(struct device *dev, 1425 struct device_attribute *attr, char *buf) 1426 { 1427 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1428 1429 return sprintf(buf, "0x%04x\n", 1430 be16_to_cpu(dcr->subsystem_revision_id)); 1431 } 1432 static DEVICE_ATTR_RO(subsystem_rev_id); 1433 1434 static ssize_t subsystem_device_show(struct device *dev, 1435 struct device_attribute *attr, char *buf) 1436 { 1437 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1438 1439 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); 1440 } 1441 static DEVICE_ATTR_RO(subsystem_device); 1442 1443 static int num_nvdimm_formats(struct nvdimm *nvdimm) 1444 { 1445 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1446 int formats = 0; 1447 1448 if (nfit_mem->memdev_pmem) 1449 formats++; 1450 if (nfit_mem->memdev_bdw) 1451 formats++; 1452 return formats; 1453 } 1454 1455 static ssize_t format_show(struct device *dev, 1456 struct device_attribute *attr, char *buf) 1457 { 1458 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1459 1460 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); 1461 } 1462 static DEVICE_ATTR_RO(format); 1463 1464 static ssize_t format1_show(struct device *dev, 1465 struct device_attribute *attr, char *buf) 1466 { 1467 u32 handle; 1468 ssize_t rc = -ENXIO; 1469 struct nfit_mem *nfit_mem; 1470 struct nfit_memdev *nfit_memdev; 1471 struct acpi_nfit_desc *acpi_desc; 1472 struct nvdimm *nvdimm = to_nvdimm(dev); 1473 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1474 1475 nfit_mem = nvdimm_provider_data(nvdimm); 1476 acpi_desc = nfit_mem->acpi_desc; 1477 handle = to_nfit_memdev(dev)->device_handle; 1478 1479 /* assumes DIMMs have at most 2 published interface codes */ 1480 mutex_lock(&acpi_desc->init_mutex); 1481 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1482 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 1483 struct nfit_dcr *nfit_dcr; 1484 1485 if (memdev->device_handle != handle) 1486 continue; 1487 1488 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1489 if (nfit_dcr->dcr->region_index != memdev->region_index) 1490 continue; 1491 if (nfit_dcr->dcr->code == dcr->code) 1492 continue; 1493 rc = sprintf(buf, "0x%04x\n", 1494 le16_to_cpu(nfit_dcr->dcr->code)); 1495 break; 1496 } 1497 if (rc != ENXIO) 1498 break; 1499 } 1500 mutex_unlock(&acpi_desc->init_mutex); 1501 return rc; 1502 } 1503 static DEVICE_ATTR_RO(format1); 1504 1505 static ssize_t formats_show(struct device *dev, 1506 struct device_attribute *attr, char *buf) 1507 { 1508 struct nvdimm *nvdimm = to_nvdimm(dev); 1509 1510 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); 1511 } 1512 static DEVICE_ATTR_RO(formats); 1513 1514 static ssize_t serial_show(struct device *dev, 1515 struct device_attribute *attr, char *buf) 1516 { 1517 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1518 1519 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); 1520 } 1521 static DEVICE_ATTR_RO(serial); 1522 1523 static ssize_t family_show(struct device *dev, 1524 struct device_attribute *attr, char *buf) 1525 { 1526 struct nvdimm *nvdimm = to_nvdimm(dev); 1527 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1528 1529 if (nfit_mem->family < 0) 1530 return -ENXIO; 1531 return sprintf(buf, "%d\n", nfit_mem->family); 1532 } 1533 static DEVICE_ATTR_RO(family); 1534 1535 static ssize_t dsm_mask_show(struct device *dev, 1536 struct device_attribute *attr, char *buf) 1537 { 1538 struct nvdimm *nvdimm = to_nvdimm(dev); 1539 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1540 1541 if (nfit_mem->family < 0) 1542 return -ENXIO; 1543 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); 1544 } 1545 static DEVICE_ATTR_RO(dsm_mask); 1546 1547 static ssize_t flags_show(struct device *dev, 1548 struct device_attribute *attr, char *buf) 1549 { 1550 u16 flags = to_nfit_memdev(dev)->flags; 1551 1552 return sprintf(buf, "%s%s%s%s%s%s%s\n", 1553 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", 1554 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", 1555 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", 1556 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", 1557 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "", 1558 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "", 1559 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : ""); 1560 } 1561 static DEVICE_ATTR_RO(flags); 1562 1563 static ssize_t id_show(struct device *dev, 1564 struct device_attribute *attr, char *buf) 1565 { 1566 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1567 1568 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) 1569 return sprintf(buf, "%04x-%02x-%04x-%08x\n", 1570 be16_to_cpu(dcr->vendor_id), 1571 dcr->manufacturing_location, 1572 be16_to_cpu(dcr->manufacturing_date), 1573 be32_to_cpu(dcr->serial_number)); 1574 else 1575 return sprintf(buf, "%04x-%08x\n", 1576 be16_to_cpu(dcr->vendor_id), 1577 be32_to_cpu(dcr->serial_number)); 1578 } 1579 static DEVICE_ATTR_RO(id); 1580 1581 static struct attribute *acpi_nfit_dimm_attributes[] = { 1582 &dev_attr_handle.attr, 1583 &dev_attr_phys_id.attr, 1584 &dev_attr_vendor.attr, 1585 &dev_attr_device.attr, 1586 &dev_attr_rev_id.attr, 1587 &dev_attr_subsystem_vendor.attr, 1588 &dev_attr_subsystem_device.attr, 1589 &dev_attr_subsystem_rev_id.attr, 1590 &dev_attr_format.attr, 1591 &dev_attr_formats.attr, 1592 &dev_attr_format1.attr, 1593 &dev_attr_serial.attr, 1594 &dev_attr_flags.attr, 1595 &dev_attr_id.attr, 1596 &dev_attr_family.attr, 1597 &dev_attr_dsm_mask.attr, 1598 NULL, 1599 }; 1600 1601 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, 1602 struct attribute *a, int n) 1603 { 1604 struct device *dev = container_of(kobj, struct device, kobj); 1605 struct nvdimm *nvdimm = to_nvdimm(dev); 1606 1607 if (!to_nfit_dcr(dev)) { 1608 /* Without a dcr only the memdev attributes can be surfaced */ 1609 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr 1610 || a == &dev_attr_flags.attr 1611 || a == &dev_attr_family.attr 1612 || a == &dev_attr_dsm_mask.attr) 1613 return a->mode; 1614 return 0; 1615 } 1616 1617 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) 1618 return 0; 1619 return a->mode; 1620 } 1621 1622 static const struct attribute_group acpi_nfit_dimm_attribute_group = { 1623 .name = "nfit", 1624 .attrs = acpi_nfit_dimm_attributes, 1625 .is_visible = acpi_nfit_dimm_attr_visible, 1626 }; 1627 1628 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { 1629 &nvdimm_attribute_group, 1630 &nd_device_attribute_group, 1631 &acpi_nfit_dimm_attribute_group, 1632 NULL, 1633 }; 1634 1635 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, 1636 u32 device_handle) 1637 { 1638 struct nfit_mem *nfit_mem; 1639 1640 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1641 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) 1642 return nfit_mem->nvdimm; 1643 1644 return NULL; 1645 } 1646 1647 void __acpi_nvdimm_notify(struct device *dev, u32 event) 1648 { 1649 struct nfit_mem *nfit_mem; 1650 struct acpi_nfit_desc *acpi_desc; 1651 1652 dev_dbg(dev->parent, "%s: event: %d\n", dev_name(dev), 1653 event); 1654 1655 if (event != NFIT_NOTIFY_DIMM_HEALTH) { 1656 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev), 1657 event); 1658 return; 1659 } 1660 1661 acpi_desc = dev_get_drvdata(dev->parent); 1662 if (!acpi_desc) 1663 return; 1664 1665 /* 1666 * If we successfully retrieved acpi_desc, then we know nfit_mem data 1667 * is still valid. 1668 */ 1669 nfit_mem = dev_get_drvdata(dev); 1670 if (nfit_mem && nfit_mem->flags_attr) 1671 sysfs_notify_dirent(nfit_mem->flags_attr); 1672 } 1673 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify); 1674 1675 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data) 1676 { 1677 struct acpi_device *adev = data; 1678 struct device *dev = &adev->dev; 1679 1680 device_lock(dev->parent); 1681 __acpi_nvdimm_notify(dev, event); 1682 device_unlock(dev->parent); 1683 } 1684 1685 static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method) 1686 { 1687 acpi_handle handle; 1688 acpi_status status; 1689 1690 status = acpi_get_handle(adev->handle, method, &handle); 1691 1692 if (ACPI_SUCCESS(status)) 1693 return true; 1694 return false; 1695 } 1696 1697 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, 1698 struct nfit_mem *nfit_mem, u32 device_handle) 1699 { 1700 struct acpi_device *adev, *adev_dimm; 1701 struct device *dev = acpi_desc->dev; 1702 unsigned long dsm_mask, label_mask; 1703 const guid_t *guid; 1704 int i; 1705 int family = -1; 1706 1707 /* nfit test assumes 1:1 relationship between commands and dsms */ 1708 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; 1709 nfit_mem->family = NVDIMM_FAMILY_INTEL; 1710 adev = to_acpi_dev(acpi_desc); 1711 if (!adev) 1712 return 0; 1713 1714 adev_dimm = acpi_find_child_device(adev, device_handle, false); 1715 nfit_mem->adev = adev_dimm; 1716 if (!adev_dimm) { 1717 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", 1718 device_handle); 1719 return force_enable_dimms ? 0 : -ENODEV; 1720 } 1721 1722 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle, 1723 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) { 1724 dev_err(dev, "%s: notification registration failed\n", 1725 dev_name(&adev_dimm->dev)); 1726 return -ENXIO; 1727 } 1728 /* 1729 * Record nfit_mem for the notification path to track back to 1730 * the nfit sysfs attributes for this dimm device object. 1731 */ 1732 dev_set_drvdata(&adev_dimm->dev, nfit_mem); 1733 1734 /* 1735 * Until standardization materializes we need to consider 4 1736 * different command sets. Note, that checking for function0 (bit0) 1737 * tells us if any commands are reachable through this GUID. 1738 */ 1739 for (i = 0; i <= NVDIMM_FAMILY_MAX; i++) 1740 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) 1741 if (family < 0 || i == default_dsm_family) 1742 family = i; 1743 1744 /* limit the supported commands to those that are publicly documented */ 1745 nfit_mem->family = family; 1746 if (override_dsm_mask && !disable_vendor_specific) 1747 dsm_mask = override_dsm_mask; 1748 else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1749 dsm_mask = NVDIMM_INTEL_CMDMASK; 1750 if (disable_vendor_specific) 1751 dsm_mask &= ~(1 << ND_CMD_VENDOR); 1752 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { 1753 dsm_mask = 0x1c3c76; 1754 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { 1755 dsm_mask = 0x1fe; 1756 if (disable_vendor_specific) 1757 dsm_mask &= ~(1 << 8); 1758 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { 1759 dsm_mask = 0xffffffff; 1760 } else { 1761 dev_dbg(dev, "unknown dimm command family\n"); 1762 nfit_mem->family = -1; 1763 /* DSMs are optional, continue loading the driver... */ 1764 return 0; 1765 } 1766 1767 guid = to_nfit_uuid(nfit_mem->family); 1768 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1769 if (acpi_check_dsm(adev_dimm->handle, guid, 1770 nfit_dsm_revid(nfit_mem->family, i), 1771 1ULL << i)) 1772 set_bit(i, &nfit_mem->dsm_mask); 1773 1774 /* 1775 * Prefer the NVDIMM_FAMILY_INTEL label read commands if present 1776 * due to their better semantics handling locked capacity. 1777 */ 1778 label_mask = 1 << ND_CMD_GET_CONFIG_SIZE | 1 << ND_CMD_GET_CONFIG_DATA 1779 | 1 << ND_CMD_SET_CONFIG_DATA; 1780 if (family == NVDIMM_FAMILY_INTEL 1781 && (dsm_mask & label_mask) == label_mask) 1782 return 0; 1783 1784 if (acpi_nvdimm_has_method(adev_dimm, "_LSI") 1785 && acpi_nvdimm_has_method(adev_dimm, "_LSR")) { 1786 dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev)); 1787 nfit_mem->has_lsr = true; 1788 } 1789 1790 if (nfit_mem->has_lsr && acpi_nvdimm_has_method(adev_dimm, "_LSW")) { 1791 dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev)); 1792 nfit_mem->has_lsw = true; 1793 } 1794 1795 return 0; 1796 } 1797 1798 static void shutdown_dimm_notify(void *data) 1799 { 1800 struct acpi_nfit_desc *acpi_desc = data; 1801 struct nfit_mem *nfit_mem; 1802 1803 mutex_lock(&acpi_desc->init_mutex); 1804 /* 1805 * Clear out the nfit_mem->flags_attr and shut down dimm event 1806 * notifications. 1807 */ 1808 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1809 struct acpi_device *adev_dimm = nfit_mem->adev; 1810 1811 if (nfit_mem->flags_attr) { 1812 sysfs_put(nfit_mem->flags_attr); 1813 nfit_mem->flags_attr = NULL; 1814 } 1815 if (adev_dimm) { 1816 acpi_remove_notify_handler(adev_dimm->handle, 1817 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify); 1818 dev_set_drvdata(&adev_dimm->dev, NULL); 1819 } 1820 } 1821 mutex_unlock(&acpi_desc->init_mutex); 1822 } 1823 1824 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) 1825 { 1826 struct nfit_mem *nfit_mem; 1827 int dimm_count = 0, rc; 1828 struct nvdimm *nvdimm; 1829 1830 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1831 struct acpi_nfit_flush_address *flush; 1832 unsigned long flags = 0, cmd_mask; 1833 struct nfit_memdev *nfit_memdev; 1834 u32 device_handle; 1835 u16 mem_flags; 1836 1837 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 1838 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); 1839 if (nvdimm) { 1840 dimm_count++; 1841 continue; 1842 } 1843 1844 if (nfit_mem->bdw && nfit_mem->memdev_pmem) 1845 set_bit(NDD_ALIASING, &flags); 1846 1847 /* collate flags across all memdevs for this dimm */ 1848 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1849 struct acpi_nfit_memory_map *dimm_memdev; 1850 1851 dimm_memdev = __to_nfit_memdev(nfit_mem); 1852 if (dimm_memdev->device_handle 1853 != nfit_memdev->memdev->device_handle) 1854 continue; 1855 dimm_memdev->flags |= nfit_memdev->memdev->flags; 1856 } 1857 1858 mem_flags = __to_nfit_memdev(nfit_mem)->flags; 1859 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) 1860 set_bit(NDD_UNARMED, &flags); 1861 1862 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); 1863 if (rc) 1864 continue; 1865 1866 /* 1867 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL 1868 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the 1869 * userspace interface. 1870 */ 1871 cmd_mask = 1UL << ND_CMD_CALL; 1872 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1873 /* 1874 * These commands have a 1:1 correspondence 1875 * between DSM payload and libnvdimm ioctl 1876 * payload format. 1877 */ 1878 cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK; 1879 } 1880 1881 if (nfit_mem->has_lsr) { 1882 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask); 1883 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask); 1884 } 1885 if (nfit_mem->has_lsw) 1886 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask); 1887 1888 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush 1889 : NULL; 1890 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, 1891 acpi_nfit_dimm_attribute_groups, 1892 flags, cmd_mask, flush ? flush->hint_count : 0, 1893 nfit_mem->flush_wpq); 1894 if (!nvdimm) 1895 return -ENOMEM; 1896 1897 nfit_mem->nvdimm = nvdimm; 1898 dimm_count++; 1899 1900 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) 1901 continue; 1902 1903 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n", 1904 nvdimm_name(nvdimm), 1905 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", 1906 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", 1907 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", 1908 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "", 1909 mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : ""); 1910 1911 } 1912 1913 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); 1914 if (rc) 1915 return rc; 1916 1917 /* 1918 * Now that dimms are successfully registered, and async registration 1919 * is flushed, attempt to enable event notification. 1920 */ 1921 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1922 struct kernfs_node *nfit_kernfs; 1923 1924 nvdimm = nfit_mem->nvdimm; 1925 if (!nvdimm) 1926 continue; 1927 1928 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit"); 1929 if (nfit_kernfs) 1930 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs, 1931 "flags"); 1932 sysfs_put(nfit_kernfs); 1933 if (!nfit_mem->flags_attr) 1934 dev_warn(acpi_desc->dev, "%s: notifications disabled\n", 1935 nvdimm_name(nvdimm)); 1936 } 1937 1938 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify, 1939 acpi_desc); 1940 } 1941 1942 /* 1943 * These constants are private because there are no kernel consumers of 1944 * these commands. 1945 */ 1946 enum nfit_aux_cmds { 1947 NFIT_CMD_TRANSLATE_SPA = 5, 1948 NFIT_CMD_ARS_INJECT_SET = 7, 1949 NFIT_CMD_ARS_INJECT_CLEAR = 8, 1950 NFIT_CMD_ARS_INJECT_GET = 9, 1951 }; 1952 1953 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) 1954 { 1955 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1956 const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS); 1957 struct acpi_device *adev; 1958 unsigned long dsm_mask; 1959 int i; 1960 1961 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; 1962 nd_desc->bus_dsm_mask = acpi_desc->bus_nfit_cmd_force_en; 1963 adev = to_acpi_dev(acpi_desc); 1964 if (!adev) 1965 return; 1966 1967 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) 1968 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1969 set_bit(i, &nd_desc->cmd_mask); 1970 set_bit(ND_CMD_CALL, &nd_desc->cmd_mask); 1971 1972 dsm_mask = 1973 (1 << ND_CMD_ARS_CAP) | 1974 (1 << ND_CMD_ARS_START) | 1975 (1 << ND_CMD_ARS_STATUS) | 1976 (1 << ND_CMD_CLEAR_ERROR) | 1977 (1 << NFIT_CMD_TRANSLATE_SPA) | 1978 (1 << NFIT_CMD_ARS_INJECT_SET) | 1979 (1 << NFIT_CMD_ARS_INJECT_CLEAR) | 1980 (1 << NFIT_CMD_ARS_INJECT_GET); 1981 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1982 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1983 set_bit(i, &nd_desc->bus_dsm_mask); 1984 } 1985 1986 static ssize_t range_index_show(struct device *dev, 1987 struct device_attribute *attr, char *buf) 1988 { 1989 struct nd_region *nd_region = to_nd_region(dev); 1990 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1991 1992 return sprintf(buf, "%d\n", nfit_spa->spa->range_index); 1993 } 1994 static DEVICE_ATTR_RO(range_index); 1995 1996 static struct attribute *acpi_nfit_region_attributes[] = { 1997 &dev_attr_range_index.attr, 1998 NULL, 1999 }; 2000 2001 static const struct attribute_group acpi_nfit_region_attribute_group = { 2002 .name = "nfit", 2003 .attrs = acpi_nfit_region_attributes, 2004 }; 2005 2006 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { 2007 &nd_region_attribute_group, 2008 &nd_mapping_attribute_group, 2009 &nd_device_attribute_group, 2010 &nd_numa_attribute_group, 2011 &acpi_nfit_region_attribute_group, 2012 NULL, 2013 }; 2014 2015 /* enough info to uniquely specify an interleave set */ 2016 struct nfit_set_info { 2017 struct nfit_set_info_map { 2018 u64 region_offset; 2019 u32 serial_number; 2020 u32 pad; 2021 } mapping[0]; 2022 }; 2023 2024 struct nfit_set_info2 { 2025 struct nfit_set_info_map2 { 2026 u64 region_offset; 2027 u32 serial_number; 2028 u16 vendor_id; 2029 u16 manufacturing_date; 2030 u8 manufacturing_location; 2031 u8 reserved[31]; 2032 } mapping[0]; 2033 }; 2034 2035 static size_t sizeof_nfit_set_info(int num_mappings) 2036 { 2037 return sizeof(struct nfit_set_info) 2038 + num_mappings * sizeof(struct nfit_set_info_map); 2039 } 2040 2041 static size_t sizeof_nfit_set_info2(int num_mappings) 2042 { 2043 return sizeof(struct nfit_set_info2) 2044 + num_mappings * sizeof(struct nfit_set_info_map2); 2045 } 2046 2047 static int cmp_map_compat(const void *m0, const void *m1) 2048 { 2049 const struct nfit_set_info_map *map0 = m0; 2050 const struct nfit_set_info_map *map1 = m1; 2051 2052 return memcmp(&map0->region_offset, &map1->region_offset, 2053 sizeof(u64)); 2054 } 2055 2056 static int cmp_map(const void *m0, const void *m1) 2057 { 2058 const struct nfit_set_info_map *map0 = m0; 2059 const struct nfit_set_info_map *map1 = m1; 2060 2061 if (map0->region_offset < map1->region_offset) 2062 return -1; 2063 else if (map0->region_offset > map1->region_offset) 2064 return 1; 2065 return 0; 2066 } 2067 2068 static int cmp_map2(const void *m0, const void *m1) 2069 { 2070 const struct nfit_set_info_map2 *map0 = m0; 2071 const struct nfit_set_info_map2 *map1 = m1; 2072 2073 if (map0->region_offset < map1->region_offset) 2074 return -1; 2075 else if (map0->region_offset > map1->region_offset) 2076 return 1; 2077 return 0; 2078 } 2079 2080 /* Retrieve the nth entry referencing this spa */ 2081 static struct acpi_nfit_memory_map *memdev_from_spa( 2082 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) 2083 { 2084 struct nfit_memdev *nfit_memdev; 2085 2086 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) 2087 if (nfit_memdev->memdev->range_index == range_index) 2088 if (n-- == 0) 2089 return nfit_memdev->memdev; 2090 return NULL; 2091 } 2092 2093 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, 2094 struct nd_region_desc *ndr_desc, 2095 struct acpi_nfit_system_address *spa) 2096 { 2097 struct device *dev = acpi_desc->dev; 2098 struct nd_interleave_set *nd_set; 2099 u16 nr = ndr_desc->num_mappings; 2100 struct nfit_set_info2 *info2; 2101 struct nfit_set_info *info; 2102 int i; 2103 2104 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 2105 if (!nd_set) 2106 return -ENOMEM; 2107 ndr_desc->nd_set = nd_set; 2108 guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid); 2109 2110 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); 2111 if (!info) 2112 return -ENOMEM; 2113 2114 info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL); 2115 if (!info2) 2116 return -ENOMEM; 2117 2118 for (i = 0; i < nr; i++) { 2119 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 2120 struct nfit_set_info_map *map = &info->mapping[i]; 2121 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2122 struct nvdimm *nvdimm = mapping->nvdimm; 2123 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2124 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, 2125 spa->range_index, i); 2126 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2127 2128 if (!memdev || !nfit_mem->dcr) { 2129 dev_err(dev, "%s: failed to find DCR\n", __func__); 2130 return -ENODEV; 2131 } 2132 2133 map->region_offset = memdev->region_offset; 2134 map->serial_number = dcr->serial_number; 2135 2136 map2->region_offset = memdev->region_offset; 2137 map2->serial_number = dcr->serial_number; 2138 map2->vendor_id = dcr->vendor_id; 2139 map2->manufacturing_date = dcr->manufacturing_date; 2140 map2->manufacturing_location = dcr->manufacturing_location; 2141 } 2142 2143 /* v1.1 namespaces */ 2144 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2145 cmp_map, NULL); 2146 nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2147 2148 /* v1.2 namespaces */ 2149 sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2), 2150 cmp_map2, NULL); 2151 nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0); 2152 2153 /* support v1.1 namespaces created with the wrong sort order */ 2154 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2155 cmp_map_compat, NULL); 2156 nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2157 2158 /* record the result of the sort for the mapping position */ 2159 for (i = 0; i < nr; i++) { 2160 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2161 int j; 2162 2163 for (j = 0; j < nr; j++) { 2164 struct nd_mapping_desc *mapping = &ndr_desc->mapping[j]; 2165 struct nvdimm *nvdimm = mapping->nvdimm; 2166 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2167 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2168 2169 if (map2->serial_number == dcr->serial_number && 2170 map2->vendor_id == dcr->vendor_id && 2171 map2->manufacturing_date == dcr->manufacturing_date && 2172 map2->manufacturing_location 2173 == dcr->manufacturing_location) { 2174 mapping->position = i; 2175 break; 2176 } 2177 } 2178 } 2179 2180 ndr_desc->nd_set = nd_set; 2181 devm_kfree(dev, info); 2182 devm_kfree(dev, info2); 2183 2184 return 0; 2185 } 2186 2187 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) 2188 { 2189 struct acpi_nfit_interleave *idt = mmio->idt; 2190 u32 sub_line_offset, line_index, line_offset; 2191 u64 line_no, table_skip_count, table_offset; 2192 2193 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); 2194 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); 2195 line_offset = idt->line_offset[line_index] 2196 * mmio->line_size; 2197 table_offset = table_skip_count * mmio->table_size; 2198 2199 return mmio->base_offset + line_offset + table_offset + sub_line_offset; 2200 } 2201 2202 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) 2203 { 2204 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2205 u64 offset = nfit_blk->stat_offset + mmio->size * bw; 2206 const u32 STATUS_MASK = 0x80000037; 2207 2208 if (mmio->num_lines) 2209 offset = to_interleave_offset(offset, mmio); 2210 2211 return readl(mmio->addr.base + offset) & STATUS_MASK; 2212 } 2213 2214 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, 2215 resource_size_t dpa, unsigned int len, unsigned int write) 2216 { 2217 u64 cmd, offset; 2218 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2219 2220 enum { 2221 BCW_OFFSET_MASK = (1ULL << 48)-1, 2222 BCW_LEN_SHIFT = 48, 2223 BCW_LEN_MASK = (1ULL << 8) - 1, 2224 BCW_CMD_SHIFT = 56, 2225 }; 2226 2227 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; 2228 len = len >> L1_CACHE_SHIFT; 2229 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; 2230 cmd |= ((u64) write) << BCW_CMD_SHIFT; 2231 2232 offset = nfit_blk->cmd_offset + mmio->size * bw; 2233 if (mmio->num_lines) 2234 offset = to_interleave_offset(offset, mmio); 2235 2236 writeq(cmd, mmio->addr.base + offset); 2237 nvdimm_flush(nfit_blk->nd_region); 2238 2239 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) 2240 readq(mmio->addr.base + offset); 2241 } 2242 2243 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 2244 resource_size_t dpa, void *iobuf, size_t len, int rw, 2245 unsigned int lane) 2246 { 2247 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2248 unsigned int copied = 0; 2249 u64 base_offset; 2250 int rc; 2251 2252 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES 2253 + lane * mmio->size; 2254 write_blk_ctl(nfit_blk, lane, dpa, len, rw); 2255 while (len) { 2256 unsigned int c; 2257 u64 offset; 2258 2259 if (mmio->num_lines) { 2260 u32 line_offset; 2261 2262 offset = to_interleave_offset(base_offset + copied, 2263 mmio); 2264 div_u64_rem(offset, mmio->line_size, &line_offset); 2265 c = min_t(size_t, len, mmio->line_size - line_offset); 2266 } else { 2267 offset = base_offset + nfit_blk->bdw_offset; 2268 c = len; 2269 } 2270 2271 if (rw) 2272 memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c); 2273 else { 2274 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) 2275 arch_invalidate_pmem((void __force *) 2276 mmio->addr.aperture + offset, c); 2277 2278 memcpy(iobuf + copied, mmio->addr.aperture + offset, c); 2279 } 2280 2281 copied += c; 2282 len -= c; 2283 } 2284 2285 if (rw) 2286 nvdimm_flush(nfit_blk->nd_region); 2287 2288 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; 2289 return rc; 2290 } 2291 2292 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, 2293 resource_size_t dpa, void *iobuf, u64 len, int rw) 2294 { 2295 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); 2296 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2297 struct nd_region *nd_region = nfit_blk->nd_region; 2298 unsigned int lane, copied = 0; 2299 int rc = 0; 2300 2301 lane = nd_region_acquire_lane(nd_region); 2302 while (len) { 2303 u64 c = min(len, mmio->size); 2304 2305 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, 2306 iobuf + copied, c, rw, lane); 2307 if (rc) 2308 break; 2309 2310 copied += c; 2311 len -= c; 2312 } 2313 nd_region_release_lane(nd_region, lane); 2314 2315 return rc; 2316 } 2317 2318 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, 2319 struct acpi_nfit_interleave *idt, u16 interleave_ways) 2320 { 2321 if (idt) { 2322 mmio->num_lines = idt->line_count; 2323 mmio->line_size = idt->line_size; 2324 if (interleave_ways == 0) 2325 return -ENXIO; 2326 mmio->table_size = mmio->num_lines * interleave_ways 2327 * mmio->line_size; 2328 } 2329 2330 return 0; 2331 } 2332 2333 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, 2334 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) 2335 { 2336 struct nd_cmd_dimm_flags flags; 2337 int rc; 2338 2339 memset(&flags, 0, sizeof(flags)); 2340 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, 2341 sizeof(flags), NULL); 2342 2343 if (rc >= 0 && flags.status == 0) 2344 nfit_blk->dimm_flags = flags.flags; 2345 else if (rc == -ENOTTY) { 2346 /* fall back to a conservative default */ 2347 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; 2348 rc = 0; 2349 } else 2350 rc = -ENXIO; 2351 2352 return rc; 2353 } 2354 2355 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, 2356 struct device *dev) 2357 { 2358 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 2359 struct nd_blk_region *ndbr = to_nd_blk_region(dev); 2360 struct nfit_blk_mmio *mmio; 2361 struct nfit_blk *nfit_blk; 2362 struct nfit_mem *nfit_mem; 2363 struct nvdimm *nvdimm; 2364 int rc; 2365 2366 nvdimm = nd_blk_region_to_dimm(ndbr); 2367 nfit_mem = nvdimm_provider_data(nvdimm); 2368 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { 2369 dev_dbg(dev, "missing%s%s%s\n", 2370 nfit_mem ? "" : " nfit_mem", 2371 (nfit_mem && nfit_mem->dcr) ? "" : " dcr", 2372 (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); 2373 return -ENXIO; 2374 } 2375 2376 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); 2377 if (!nfit_blk) 2378 return -ENOMEM; 2379 nd_blk_region_set_provider_data(ndbr, nfit_blk); 2380 nfit_blk->nd_region = to_nd_region(dev); 2381 2382 /* map block aperture memory */ 2383 nfit_blk->bdw_offset = nfit_mem->bdw->offset; 2384 mmio = &nfit_blk->mmio[BDW]; 2385 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, 2386 nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr)); 2387 if (!mmio->addr.base) { 2388 dev_dbg(dev, "%s failed to map bdw\n", 2389 nvdimm_name(nvdimm)); 2390 return -ENOMEM; 2391 } 2392 mmio->size = nfit_mem->bdw->size; 2393 mmio->base_offset = nfit_mem->memdev_bdw->region_offset; 2394 mmio->idt = nfit_mem->idt_bdw; 2395 mmio->spa = nfit_mem->spa_bdw; 2396 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, 2397 nfit_mem->memdev_bdw->interleave_ways); 2398 if (rc) { 2399 dev_dbg(dev, "%s failed to init bdw interleave\n", 2400 nvdimm_name(nvdimm)); 2401 return rc; 2402 } 2403 2404 /* map block control memory */ 2405 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; 2406 nfit_blk->stat_offset = nfit_mem->dcr->status_offset; 2407 mmio = &nfit_blk->mmio[DCR]; 2408 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, 2409 nfit_mem->spa_dcr->length); 2410 if (!mmio->addr.base) { 2411 dev_dbg(dev, "%s failed to map dcr\n", 2412 nvdimm_name(nvdimm)); 2413 return -ENOMEM; 2414 } 2415 mmio->size = nfit_mem->dcr->window_size; 2416 mmio->base_offset = nfit_mem->memdev_dcr->region_offset; 2417 mmio->idt = nfit_mem->idt_dcr; 2418 mmio->spa = nfit_mem->spa_dcr; 2419 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, 2420 nfit_mem->memdev_dcr->interleave_ways); 2421 if (rc) { 2422 dev_dbg(dev, "%s failed to init dcr interleave\n", 2423 nvdimm_name(nvdimm)); 2424 return rc; 2425 } 2426 2427 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); 2428 if (rc < 0) { 2429 dev_dbg(dev, "%s failed get DIMM flags\n", 2430 nvdimm_name(nvdimm)); 2431 return rc; 2432 } 2433 2434 if (nvdimm_has_flush(nfit_blk->nd_region) < 0) 2435 dev_warn(dev, "unable to guarantee persistence of writes\n"); 2436 2437 if (mmio->line_size == 0) 2438 return 0; 2439 2440 if ((u32) nfit_blk->cmd_offset % mmio->line_size 2441 + 8 > mmio->line_size) { 2442 dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); 2443 return -ENXIO; 2444 } else if ((u32) nfit_blk->stat_offset % mmio->line_size 2445 + 8 > mmio->line_size) { 2446 dev_dbg(dev, "stat_offset crosses interleave boundary\n"); 2447 return -ENXIO; 2448 } 2449 2450 return 0; 2451 } 2452 2453 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, 2454 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) 2455 { 2456 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2457 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2458 int cmd_rc, rc; 2459 2460 cmd->address = spa->address; 2461 cmd->length = spa->length; 2462 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, 2463 sizeof(*cmd), &cmd_rc); 2464 if (rc < 0) 2465 return rc; 2466 return cmd_rc; 2467 } 2468 2469 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) 2470 { 2471 int rc; 2472 int cmd_rc; 2473 struct nd_cmd_ars_start ars_start; 2474 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2475 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2476 2477 memset(&ars_start, 0, sizeof(ars_start)); 2478 ars_start.address = spa->address; 2479 ars_start.length = spa->length; 2480 if (test_bit(ARS_SHORT, &nfit_spa->ars_state)) 2481 ars_start.flags = ND_ARS_RETURN_PREV_DATA; 2482 if (nfit_spa_type(spa) == NFIT_SPA_PM) 2483 ars_start.type = ND_ARS_PERSISTENT; 2484 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) 2485 ars_start.type = ND_ARS_VOLATILE; 2486 else 2487 return -ENOTTY; 2488 2489 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2490 sizeof(ars_start), &cmd_rc); 2491 2492 if (rc < 0) 2493 return rc; 2494 return cmd_rc; 2495 } 2496 2497 static int ars_continue(struct acpi_nfit_desc *acpi_desc) 2498 { 2499 int rc, cmd_rc; 2500 struct nd_cmd_ars_start ars_start; 2501 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2502 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2503 2504 memset(&ars_start, 0, sizeof(ars_start)); 2505 ars_start.address = ars_status->restart_address; 2506 ars_start.length = ars_status->restart_length; 2507 ars_start.type = ars_status->type; 2508 ars_start.flags = acpi_desc->ars_start_flags; 2509 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2510 sizeof(ars_start), &cmd_rc); 2511 if (rc < 0) 2512 return rc; 2513 return cmd_rc; 2514 } 2515 2516 static int ars_get_status(struct acpi_nfit_desc *acpi_desc) 2517 { 2518 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2519 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2520 int rc, cmd_rc; 2521 2522 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, 2523 acpi_desc->max_ars, &cmd_rc); 2524 if (rc < 0) 2525 return rc; 2526 return cmd_rc; 2527 } 2528 2529 static void ars_complete(struct acpi_nfit_desc *acpi_desc, 2530 struct nfit_spa *nfit_spa) 2531 { 2532 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2533 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2534 struct nd_region *nd_region = nfit_spa->nd_region; 2535 struct device *dev; 2536 2537 if ((ars_status->address >= spa->address && ars_status->address 2538 < spa->address + spa->length) 2539 || (ars_status->address < spa->address)) { 2540 /* 2541 * Assume that if a scrub starts at an offset from the 2542 * start of nfit_spa that we are in the continuation 2543 * case. 2544 * 2545 * Otherwise, if the scrub covers the spa range, mark 2546 * any pending request complete. 2547 */ 2548 if (ars_status->address + ars_status->length 2549 >= spa->address + spa->length) 2550 /* complete */; 2551 else 2552 return; 2553 } else 2554 return; 2555 2556 if (test_bit(ARS_DONE, &nfit_spa->ars_state)) 2557 return; 2558 2559 if (!test_and_clear_bit(ARS_REQ, &nfit_spa->ars_state)) 2560 return; 2561 2562 if (nd_region) { 2563 dev = nd_region_dev(nd_region); 2564 nvdimm_region_notify(nd_region, NVDIMM_REVALIDATE_POISON); 2565 } else 2566 dev = acpi_desc->dev; 2567 2568 dev_dbg(dev, "ARS: range %d %s complete\n", spa->range_index, 2569 test_bit(ARS_SHORT, &nfit_spa->ars_state) 2570 ? "short" : "long"); 2571 clear_bit(ARS_SHORT, &nfit_spa->ars_state); 2572 if (test_and_clear_bit(ARS_REQ_REDO, &nfit_spa->ars_state)) { 2573 set_bit(ARS_SHORT, &nfit_spa->ars_state); 2574 set_bit(ARS_REQ, &nfit_spa->ars_state); 2575 dev_dbg(dev, "ARS: processing scrub request received while in progress\n"); 2576 } else 2577 set_bit(ARS_DONE, &nfit_spa->ars_state); 2578 } 2579 2580 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc) 2581 { 2582 struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus; 2583 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2584 int rc; 2585 u32 i; 2586 2587 /* 2588 * First record starts at 44 byte offset from the start of the 2589 * payload. 2590 */ 2591 if (ars_status->out_length < 44) 2592 return 0; 2593 for (i = 0; i < ars_status->num_records; i++) { 2594 /* only process full records */ 2595 if (ars_status->out_length 2596 < 44 + sizeof(struct nd_ars_record) * (i + 1)) 2597 break; 2598 rc = nvdimm_bus_add_badrange(nvdimm_bus, 2599 ars_status->records[i].err_address, 2600 ars_status->records[i].length); 2601 if (rc) 2602 return rc; 2603 } 2604 if (i < ars_status->num_records) 2605 dev_warn(acpi_desc->dev, "detected truncated ars results\n"); 2606 2607 return 0; 2608 } 2609 2610 static void acpi_nfit_remove_resource(void *data) 2611 { 2612 struct resource *res = data; 2613 2614 remove_resource(res); 2615 } 2616 2617 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, 2618 struct nd_region_desc *ndr_desc) 2619 { 2620 struct resource *res, *nd_res = ndr_desc->res; 2621 int is_pmem, ret; 2622 2623 /* No operation if the region is already registered as PMEM */ 2624 is_pmem = region_intersects(nd_res->start, resource_size(nd_res), 2625 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); 2626 if (is_pmem == REGION_INTERSECTS) 2627 return 0; 2628 2629 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); 2630 if (!res) 2631 return -ENOMEM; 2632 2633 res->name = "Persistent Memory"; 2634 res->start = nd_res->start; 2635 res->end = nd_res->end; 2636 res->flags = IORESOURCE_MEM; 2637 res->desc = IORES_DESC_PERSISTENT_MEMORY; 2638 2639 ret = insert_resource(&iomem_resource, res); 2640 if (ret) 2641 return ret; 2642 2643 ret = devm_add_action_or_reset(acpi_desc->dev, 2644 acpi_nfit_remove_resource, 2645 res); 2646 if (ret) 2647 return ret; 2648 2649 return 0; 2650 } 2651 2652 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, 2653 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc, 2654 struct acpi_nfit_memory_map *memdev, 2655 struct nfit_spa *nfit_spa) 2656 { 2657 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, 2658 memdev->device_handle); 2659 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2660 struct nd_blk_region_desc *ndbr_desc; 2661 struct nfit_mem *nfit_mem; 2662 int rc; 2663 2664 if (!nvdimm) { 2665 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", 2666 spa->range_index, memdev->device_handle); 2667 return -ENODEV; 2668 } 2669 2670 mapping->nvdimm = nvdimm; 2671 switch (nfit_spa_type(spa)) { 2672 case NFIT_SPA_PM: 2673 case NFIT_SPA_VOLATILE: 2674 mapping->start = memdev->address; 2675 mapping->size = memdev->region_size; 2676 break; 2677 case NFIT_SPA_DCR: 2678 nfit_mem = nvdimm_provider_data(nvdimm); 2679 if (!nfit_mem || !nfit_mem->bdw) { 2680 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", 2681 spa->range_index, nvdimm_name(nvdimm)); 2682 break; 2683 } 2684 2685 mapping->size = nfit_mem->bdw->capacity; 2686 mapping->start = nfit_mem->bdw->start_address; 2687 ndr_desc->num_lanes = nfit_mem->bdw->windows; 2688 ndr_desc->mapping = mapping; 2689 ndr_desc->num_mappings = 1; 2690 ndbr_desc = to_blk_region_desc(ndr_desc); 2691 ndbr_desc->enable = acpi_nfit_blk_region_enable; 2692 ndbr_desc->do_io = acpi_desc->blk_do_io; 2693 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2694 if (rc) 2695 return rc; 2696 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, 2697 ndr_desc); 2698 if (!nfit_spa->nd_region) 2699 return -ENOMEM; 2700 break; 2701 } 2702 2703 return 0; 2704 } 2705 2706 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) 2707 { 2708 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2709 nfit_spa_type(spa) == NFIT_SPA_VCD || 2710 nfit_spa_type(spa) == NFIT_SPA_PDISK || 2711 nfit_spa_type(spa) == NFIT_SPA_PCD); 2712 } 2713 2714 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa) 2715 { 2716 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2717 nfit_spa_type(spa) == NFIT_SPA_VCD || 2718 nfit_spa_type(spa) == NFIT_SPA_VOLATILE); 2719 } 2720 2721 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, 2722 struct nfit_spa *nfit_spa) 2723 { 2724 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS]; 2725 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2726 struct nd_blk_region_desc ndbr_desc; 2727 struct nd_region_desc *ndr_desc; 2728 struct nfit_memdev *nfit_memdev; 2729 struct nvdimm_bus *nvdimm_bus; 2730 struct resource res; 2731 int count = 0, rc; 2732 2733 if (nfit_spa->nd_region) 2734 return 0; 2735 2736 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { 2737 dev_dbg(acpi_desc->dev, "detected invalid spa index\n"); 2738 return 0; 2739 } 2740 2741 memset(&res, 0, sizeof(res)); 2742 memset(&mappings, 0, sizeof(mappings)); 2743 memset(&ndbr_desc, 0, sizeof(ndbr_desc)); 2744 res.start = spa->address; 2745 res.end = res.start + spa->length - 1; 2746 ndr_desc = &ndbr_desc.ndr_desc; 2747 ndr_desc->res = &res; 2748 ndr_desc->provider_data = nfit_spa; 2749 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; 2750 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) 2751 ndr_desc->numa_node = acpi_map_pxm_to_online_node( 2752 spa->proximity_domain); 2753 else 2754 ndr_desc->numa_node = NUMA_NO_NODE; 2755 2756 /* 2757 * Persistence domain bits are hierarchical, if 2758 * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then 2759 * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied. 2760 */ 2761 if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH) 2762 set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags); 2763 else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH) 2764 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags); 2765 2766 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 2767 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 2768 struct nd_mapping_desc *mapping; 2769 2770 if (memdev->range_index != spa->range_index) 2771 continue; 2772 if (count >= ND_MAX_MAPPINGS) { 2773 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", 2774 spa->range_index, ND_MAX_MAPPINGS); 2775 return -ENXIO; 2776 } 2777 mapping = &mappings[count++]; 2778 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc, 2779 memdev, nfit_spa); 2780 if (rc) 2781 goto out; 2782 } 2783 2784 ndr_desc->mapping = mappings; 2785 ndr_desc->num_mappings = count; 2786 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2787 if (rc) 2788 goto out; 2789 2790 nvdimm_bus = acpi_desc->nvdimm_bus; 2791 if (nfit_spa_type(spa) == NFIT_SPA_PM) { 2792 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); 2793 if (rc) { 2794 dev_warn(acpi_desc->dev, 2795 "failed to insert pmem resource to iomem: %d\n", 2796 rc); 2797 goto out; 2798 } 2799 2800 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2801 ndr_desc); 2802 if (!nfit_spa->nd_region) 2803 rc = -ENOMEM; 2804 } else if (nfit_spa_is_volatile(spa)) { 2805 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, 2806 ndr_desc); 2807 if (!nfit_spa->nd_region) 2808 rc = -ENOMEM; 2809 } else if (nfit_spa_is_virtual(spa)) { 2810 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2811 ndr_desc); 2812 if (!nfit_spa->nd_region) 2813 rc = -ENOMEM; 2814 } 2815 2816 out: 2817 if (rc) 2818 dev_err(acpi_desc->dev, "failed to register spa range %d\n", 2819 nfit_spa->spa->range_index); 2820 return rc; 2821 } 2822 2823 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc) 2824 { 2825 struct device *dev = acpi_desc->dev; 2826 struct nd_cmd_ars_status *ars_status; 2827 2828 if (acpi_desc->ars_status) { 2829 memset(acpi_desc->ars_status, 0, acpi_desc->max_ars); 2830 return 0; 2831 } 2832 2833 ars_status = devm_kzalloc(dev, acpi_desc->max_ars, GFP_KERNEL); 2834 if (!ars_status) 2835 return -ENOMEM; 2836 acpi_desc->ars_status = ars_status; 2837 return 0; 2838 } 2839 2840 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc) 2841 { 2842 int rc; 2843 2844 if (ars_status_alloc(acpi_desc)) 2845 return -ENOMEM; 2846 2847 rc = ars_get_status(acpi_desc); 2848 2849 if (rc < 0 && rc != -ENOSPC) 2850 return rc; 2851 2852 if (ars_status_process_records(acpi_desc)) 2853 return -ENOMEM; 2854 2855 return 0; 2856 } 2857 2858 static int ars_register(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa, 2859 int *query_rc) 2860 { 2861 int rc = *query_rc; 2862 2863 if (no_init_ars) 2864 return acpi_nfit_register_region(acpi_desc, nfit_spa); 2865 2866 set_bit(ARS_REQ, &nfit_spa->ars_state); 2867 set_bit(ARS_SHORT, &nfit_spa->ars_state); 2868 2869 switch (rc) { 2870 case 0: 2871 case -EAGAIN: 2872 rc = ars_start(acpi_desc, nfit_spa); 2873 if (rc == -EBUSY) { 2874 *query_rc = rc; 2875 break; 2876 } else if (rc == 0) { 2877 rc = acpi_nfit_query_poison(acpi_desc); 2878 } else { 2879 set_bit(ARS_FAILED, &nfit_spa->ars_state); 2880 break; 2881 } 2882 if (rc == -EAGAIN) 2883 clear_bit(ARS_SHORT, &nfit_spa->ars_state); 2884 else if (rc == 0) 2885 ars_complete(acpi_desc, nfit_spa); 2886 break; 2887 case -EBUSY: 2888 case -ENOSPC: 2889 break; 2890 default: 2891 set_bit(ARS_FAILED, &nfit_spa->ars_state); 2892 break; 2893 } 2894 2895 if (test_and_clear_bit(ARS_DONE, &nfit_spa->ars_state)) 2896 set_bit(ARS_REQ, &nfit_spa->ars_state); 2897 2898 return acpi_nfit_register_region(acpi_desc, nfit_spa); 2899 } 2900 2901 static void ars_complete_all(struct acpi_nfit_desc *acpi_desc) 2902 { 2903 struct nfit_spa *nfit_spa; 2904 2905 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2906 if (test_bit(ARS_FAILED, &nfit_spa->ars_state)) 2907 continue; 2908 ars_complete(acpi_desc, nfit_spa); 2909 } 2910 } 2911 2912 static unsigned int __acpi_nfit_scrub(struct acpi_nfit_desc *acpi_desc, 2913 int query_rc) 2914 { 2915 unsigned int tmo = acpi_desc->scrub_tmo; 2916 struct device *dev = acpi_desc->dev; 2917 struct nfit_spa *nfit_spa; 2918 2919 if (acpi_desc->cancel) 2920 return 0; 2921 2922 if (query_rc == -EBUSY) { 2923 dev_dbg(dev, "ARS: ARS busy\n"); 2924 return min(30U * 60U, tmo * 2); 2925 } 2926 if (query_rc == -ENOSPC) { 2927 dev_dbg(dev, "ARS: ARS continue\n"); 2928 ars_continue(acpi_desc); 2929 return 1; 2930 } 2931 if (query_rc && query_rc != -EAGAIN) { 2932 unsigned long long addr, end; 2933 2934 addr = acpi_desc->ars_status->address; 2935 end = addr + acpi_desc->ars_status->length; 2936 dev_dbg(dev, "ARS: %llx-%llx failed (%d)\n", addr, end, 2937 query_rc); 2938 } 2939 2940 ars_complete_all(acpi_desc); 2941 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2942 if (test_bit(ARS_FAILED, &nfit_spa->ars_state)) 2943 continue; 2944 if (test_bit(ARS_REQ, &nfit_spa->ars_state)) { 2945 int rc = ars_start(acpi_desc, nfit_spa); 2946 2947 clear_bit(ARS_DONE, &nfit_spa->ars_state); 2948 dev = nd_region_dev(nfit_spa->nd_region); 2949 dev_dbg(dev, "ARS: range %d ARS start (%d)\n", 2950 nfit_spa->spa->range_index, rc); 2951 if (rc == 0 || rc == -EBUSY) 2952 return 1; 2953 dev_err(dev, "ARS: range %d ARS failed (%d)\n", 2954 nfit_spa->spa->range_index, rc); 2955 set_bit(ARS_FAILED, &nfit_spa->ars_state); 2956 } 2957 } 2958 return 0; 2959 } 2960 2961 static void __sched_ars(struct acpi_nfit_desc *acpi_desc, unsigned int tmo) 2962 { 2963 lockdep_assert_held(&acpi_desc->init_mutex); 2964 2965 acpi_desc->scrub_busy = 1; 2966 /* note this should only be set from within the workqueue */ 2967 if (tmo) 2968 acpi_desc->scrub_tmo = tmo; 2969 queue_delayed_work(nfit_wq, &acpi_desc->dwork, tmo * HZ); 2970 } 2971 2972 static void sched_ars(struct acpi_nfit_desc *acpi_desc) 2973 { 2974 __sched_ars(acpi_desc, 0); 2975 } 2976 2977 static void notify_ars_done(struct acpi_nfit_desc *acpi_desc) 2978 { 2979 lockdep_assert_held(&acpi_desc->init_mutex); 2980 2981 acpi_desc->scrub_busy = 0; 2982 acpi_desc->scrub_count++; 2983 if (acpi_desc->scrub_count_state) 2984 sysfs_notify_dirent(acpi_desc->scrub_count_state); 2985 } 2986 2987 static void acpi_nfit_scrub(struct work_struct *work) 2988 { 2989 struct acpi_nfit_desc *acpi_desc; 2990 unsigned int tmo; 2991 int query_rc; 2992 2993 acpi_desc = container_of(work, typeof(*acpi_desc), dwork.work); 2994 mutex_lock(&acpi_desc->init_mutex); 2995 query_rc = acpi_nfit_query_poison(acpi_desc); 2996 tmo = __acpi_nfit_scrub(acpi_desc, query_rc); 2997 if (tmo) 2998 __sched_ars(acpi_desc, tmo); 2999 else 3000 notify_ars_done(acpi_desc); 3001 memset(acpi_desc->ars_status, 0, acpi_desc->max_ars); 3002 mutex_unlock(&acpi_desc->init_mutex); 3003 } 3004 3005 static void acpi_nfit_init_ars(struct acpi_nfit_desc *acpi_desc, 3006 struct nfit_spa *nfit_spa) 3007 { 3008 int type = nfit_spa_type(nfit_spa->spa); 3009 struct nd_cmd_ars_cap ars_cap; 3010 int rc; 3011 3012 memset(&ars_cap, 0, sizeof(ars_cap)); 3013 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); 3014 if (rc < 0) 3015 return; 3016 /* check that the supported scrub types match the spa type */ 3017 if (type == NFIT_SPA_VOLATILE && ((ars_cap.status >> 16) 3018 & ND_ARS_VOLATILE) == 0) 3019 return; 3020 if (type == NFIT_SPA_PM && ((ars_cap.status >> 16) 3021 & ND_ARS_PERSISTENT) == 0) 3022 return; 3023 3024 nfit_spa->max_ars = ars_cap.max_ars_out; 3025 nfit_spa->clear_err_unit = ars_cap.clear_err_unit; 3026 acpi_desc->max_ars = max(nfit_spa->max_ars, acpi_desc->max_ars); 3027 clear_bit(ARS_FAILED, &nfit_spa->ars_state); 3028 set_bit(ARS_REQ, &nfit_spa->ars_state); 3029 } 3030 3031 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) 3032 { 3033 struct nfit_spa *nfit_spa; 3034 int rc, query_rc; 3035 3036 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 3037 set_bit(ARS_FAILED, &nfit_spa->ars_state); 3038 switch (nfit_spa_type(nfit_spa->spa)) { 3039 case NFIT_SPA_VOLATILE: 3040 case NFIT_SPA_PM: 3041 acpi_nfit_init_ars(acpi_desc, nfit_spa); 3042 break; 3043 } 3044 } 3045 3046 /* 3047 * Reap any results that might be pending before starting new 3048 * short requests. 3049 */ 3050 query_rc = acpi_nfit_query_poison(acpi_desc); 3051 if (query_rc == 0) 3052 ars_complete_all(acpi_desc); 3053 3054 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 3055 switch (nfit_spa_type(nfit_spa->spa)) { 3056 case NFIT_SPA_VOLATILE: 3057 case NFIT_SPA_PM: 3058 /* register regions and kick off initial ARS run */ 3059 rc = ars_register(acpi_desc, nfit_spa, &query_rc); 3060 if (rc) 3061 return rc; 3062 break; 3063 case NFIT_SPA_BDW: 3064 /* nothing to register */ 3065 break; 3066 case NFIT_SPA_DCR: 3067 case NFIT_SPA_VDISK: 3068 case NFIT_SPA_VCD: 3069 case NFIT_SPA_PDISK: 3070 case NFIT_SPA_PCD: 3071 /* register known regions that don't support ARS */ 3072 rc = acpi_nfit_register_region(acpi_desc, nfit_spa); 3073 if (rc) 3074 return rc; 3075 break; 3076 default: 3077 /* don't register unknown regions */ 3078 break; 3079 } 3080 3081 sched_ars(acpi_desc); 3082 return 0; 3083 } 3084 3085 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, 3086 struct nfit_table_prev *prev) 3087 { 3088 struct device *dev = acpi_desc->dev; 3089 3090 if (!list_empty(&prev->spas) || 3091 !list_empty(&prev->memdevs) || 3092 !list_empty(&prev->dcrs) || 3093 !list_empty(&prev->bdws) || 3094 !list_empty(&prev->idts) || 3095 !list_empty(&prev->flushes)) { 3096 dev_err(dev, "new nfit deletes entries (unsupported)\n"); 3097 return -ENXIO; 3098 } 3099 return 0; 3100 } 3101 3102 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) 3103 { 3104 struct device *dev = acpi_desc->dev; 3105 struct kernfs_node *nfit; 3106 struct device *bus_dev; 3107 3108 if (!ars_supported(acpi_desc->nvdimm_bus)) 3109 return 0; 3110 3111 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3112 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); 3113 if (!nfit) { 3114 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); 3115 return -ENODEV; 3116 } 3117 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); 3118 sysfs_put(nfit); 3119 if (!acpi_desc->scrub_count_state) { 3120 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); 3121 return -ENODEV; 3122 } 3123 3124 return 0; 3125 } 3126 3127 static void acpi_nfit_unregister(void *data) 3128 { 3129 struct acpi_nfit_desc *acpi_desc = data; 3130 3131 nvdimm_bus_unregister(acpi_desc->nvdimm_bus); 3132 } 3133 3134 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) 3135 { 3136 struct device *dev = acpi_desc->dev; 3137 struct nfit_table_prev prev; 3138 const void *end; 3139 int rc; 3140 3141 if (!acpi_desc->nvdimm_bus) { 3142 acpi_nfit_init_dsms(acpi_desc); 3143 3144 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, 3145 &acpi_desc->nd_desc); 3146 if (!acpi_desc->nvdimm_bus) 3147 return -ENOMEM; 3148 3149 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister, 3150 acpi_desc); 3151 if (rc) 3152 return rc; 3153 3154 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); 3155 if (rc) 3156 return rc; 3157 3158 /* register this acpi_desc for mce notifications */ 3159 mutex_lock(&acpi_desc_lock); 3160 list_add_tail(&acpi_desc->list, &acpi_descs); 3161 mutex_unlock(&acpi_desc_lock); 3162 } 3163 3164 mutex_lock(&acpi_desc->init_mutex); 3165 3166 INIT_LIST_HEAD(&prev.spas); 3167 INIT_LIST_HEAD(&prev.memdevs); 3168 INIT_LIST_HEAD(&prev.dcrs); 3169 INIT_LIST_HEAD(&prev.bdws); 3170 INIT_LIST_HEAD(&prev.idts); 3171 INIT_LIST_HEAD(&prev.flushes); 3172 3173 list_cut_position(&prev.spas, &acpi_desc->spas, 3174 acpi_desc->spas.prev); 3175 list_cut_position(&prev.memdevs, &acpi_desc->memdevs, 3176 acpi_desc->memdevs.prev); 3177 list_cut_position(&prev.dcrs, &acpi_desc->dcrs, 3178 acpi_desc->dcrs.prev); 3179 list_cut_position(&prev.bdws, &acpi_desc->bdws, 3180 acpi_desc->bdws.prev); 3181 list_cut_position(&prev.idts, &acpi_desc->idts, 3182 acpi_desc->idts.prev); 3183 list_cut_position(&prev.flushes, &acpi_desc->flushes, 3184 acpi_desc->flushes.prev); 3185 3186 end = data + sz; 3187 while (!IS_ERR_OR_NULL(data)) 3188 data = add_table(acpi_desc, &prev, data, end); 3189 3190 if (IS_ERR(data)) { 3191 dev_dbg(dev, "nfit table parsing error: %ld\n", PTR_ERR(data)); 3192 rc = PTR_ERR(data); 3193 goto out_unlock; 3194 } 3195 3196 rc = acpi_nfit_check_deletions(acpi_desc, &prev); 3197 if (rc) 3198 goto out_unlock; 3199 3200 rc = nfit_mem_init(acpi_desc); 3201 if (rc) 3202 goto out_unlock; 3203 3204 rc = acpi_nfit_register_dimms(acpi_desc); 3205 if (rc) 3206 goto out_unlock; 3207 3208 rc = acpi_nfit_register_regions(acpi_desc); 3209 3210 out_unlock: 3211 mutex_unlock(&acpi_desc->init_mutex); 3212 return rc; 3213 } 3214 EXPORT_SYMBOL_GPL(acpi_nfit_init); 3215 3216 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) 3217 { 3218 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3219 struct device *dev = acpi_desc->dev; 3220 3221 /* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ 3222 device_lock(dev); 3223 device_unlock(dev); 3224 3225 /* Bounce the init_mutex to complete initial registration */ 3226 mutex_lock(&acpi_desc->init_mutex); 3227 mutex_unlock(&acpi_desc->init_mutex); 3228 3229 return 0; 3230 } 3231 3232 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, 3233 struct nvdimm *nvdimm, unsigned int cmd) 3234 { 3235 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3236 3237 if (nvdimm) 3238 return 0; 3239 if (cmd != ND_CMD_ARS_START) 3240 return 0; 3241 3242 /* 3243 * The kernel and userspace may race to initiate a scrub, but 3244 * the scrub thread is prepared to lose that initial race. It 3245 * just needs guarantees that any ars it initiates are not 3246 * interrupted by any intervening start reqeusts from userspace. 3247 */ 3248 if (work_busy(&acpi_desc->dwork.work)) 3249 return -EBUSY; 3250 3251 return 0; 3252 } 3253 3254 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc, unsigned long flags) 3255 { 3256 struct device *dev = acpi_desc->dev; 3257 int scheduled = 0, busy = 0; 3258 struct nfit_spa *nfit_spa; 3259 3260 mutex_lock(&acpi_desc->init_mutex); 3261 if (acpi_desc->cancel) { 3262 mutex_unlock(&acpi_desc->init_mutex); 3263 return 0; 3264 } 3265 3266 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 3267 int type = nfit_spa_type(nfit_spa->spa); 3268 3269 if (type != NFIT_SPA_PM && type != NFIT_SPA_VOLATILE) 3270 continue; 3271 if (test_bit(ARS_FAILED, &nfit_spa->ars_state)) 3272 continue; 3273 3274 if (test_and_set_bit(ARS_REQ, &nfit_spa->ars_state)) { 3275 busy++; 3276 set_bit(ARS_REQ_REDO, &nfit_spa->ars_state); 3277 } else { 3278 if (test_bit(ARS_SHORT, &flags)) 3279 set_bit(ARS_SHORT, &nfit_spa->ars_state); 3280 scheduled++; 3281 } 3282 } 3283 if (scheduled) { 3284 sched_ars(acpi_desc); 3285 dev_dbg(dev, "ars_scan triggered\n"); 3286 } 3287 mutex_unlock(&acpi_desc->init_mutex); 3288 3289 if (scheduled) 3290 return 0; 3291 if (busy) 3292 return -EBUSY; 3293 return -ENOTTY; 3294 } 3295 3296 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) 3297 { 3298 struct nvdimm_bus_descriptor *nd_desc; 3299 3300 dev_set_drvdata(dev, acpi_desc); 3301 acpi_desc->dev = dev; 3302 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; 3303 nd_desc = &acpi_desc->nd_desc; 3304 nd_desc->provider_name = "ACPI.NFIT"; 3305 nd_desc->module = THIS_MODULE; 3306 nd_desc->ndctl = acpi_nfit_ctl; 3307 nd_desc->flush_probe = acpi_nfit_flush_probe; 3308 nd_desc->clear_to_send = acpi_nfit_clear_to_send; 3309 nd_desc->attr_groups = acpi_nfit_attribute_groups; 3310 3311 INIT_LIST_HEAD(&acpi_desc->spas); 3312 INIT_LIST_HEAD(&acpi_desc->dcrs); 3313 INIT_LIST_HEAD(&acpi_desc->bdws); 3314 INIT_LIST_HEAD(&acpi_desc->idts); 3315 INIT_LIST_HEAD(&acpi_desc->flushes); 3316 INIT_LIST_HEAD(&acpi_desc->memdevs); 3317 INIT_LIST_HEAD(&acpi_desc->dimms); 3318 INIT_LIST_HEAD(&acpi_desc->list); 3319 mutex_init(&acpi_desc->init_mutex); 3320 acpi_desc->scrub_tmo = 1; 3321 INIT_DELAYED_WORK(&acpi_desc->dwork, acpi_nfit_scrub); 3322 } 3323 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); 3324 3325 static void acpi_nfit_put_table(void *table) 3326 { 3327 acpi_put_table(table); 3328 } 3329 3330 void acpi_nfit_shutdown(void *data) 3331 { 3332 struct acpi_nfit_desc *acpi_desc = data; 3333 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3334 3335 /* 3336 * Destruct under acpi_desc_lock so that nfit_handle_mce does not 3337 * race teardown 3338 */ 3339 mutex_lock(&acpi_desc_lock); 3340 list_del(&acpi_desc->list); 3341 mutex_unlock(&acpi_desc_lock); 3342 3343 mutex_lock(&acpi_desc->init_mutex); 3344 acpi_desc->cancel = 1; 3345 cancel_delayed_work_sync(&acpi_desc->dwork); 3346 mutex_unlock(&acpi_desc->init_mutex); 3347 3348 /* 3349 * Bounce the nvdimm bus lock to make sure any in-flight 3350 * acpi_nfit_ars_rescan() submissions have had a chance to 3351 * either submit or see ->cancel set. 3352 */ 3353 device_lock(bus_dev); 3354 device_unlock(bus_dev); 3355 3356 flush_workqueue(nfit_wq); 3357 } 3358 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown); 3359 3360 static int acpi_nfit_add(struct acpi_device *adev) 3361 { 3362 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3363 struct acpi_nfit_desc *acpi_desc; 3364 struct device *dev = &adev->dev; 3365 struct acpi_table_header *tbl; 3366 acpi_status status = AE_OK; 3367 acpi_size sz; 3368 int rc = 0; 3369 3370 status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl); 3371 if (ACPI_FAILURE(status)) { 3372 /* This is ok, we could have an nvdimm hotplugged later */ 3373 dev_dbg(dev, "failed to find NFIT at startup\n"); 3374 return 0; 3375 } 3376 3377 rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl); 3378 if (rc) 3379 return rc; 3380 sz = tbl->length; 3381 3382 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3383 if (!acpi_desc) 3384 return -ENOMEM; 3385 acpi_nfit_desc_init(acpi_desc, &adev->dev); 3386 3387 /* Save the acpi header for exporting the revision via sysfs */ 3388 acpi_desc->acpi_header = *tbl; 3389 3390 /* Evaluate _FIT and override with that if present */ 3391 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); 3392 if (ACPI_SUCCESS(status) && buf.length > 0) { 3393 union acpi_object *obj = buf.pointer; 3394 3395 if (obj->type == ACPI_TYPE_BUFFER) 3396 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3397 obj->buffer.length); 3398 else 3399 dev_dbg(dev, "invalid type %d, ignoring _FIT\n", 3400 (int) obj->type); 3401 kfree(buf.pointer); 3402 } else 3403 /* skip over the lead-in header table */ 3404 rc = acpi_nfit_init(acpi_desc, (void *) tbl 3405 + sizeof(struct acpi_table_nfit), 3406 sz - sizeof(struct acpi_table_nfit)); 3407 3408 if (rc) 3409 return rc; 3410 return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc); 3411 } 3412 3413 static int acpi_nfit_remove(struct acpi_device *adev) 3414 { 3415 /* see acpi_nfit_unregister */ 3416 return 0; 3417 } 3418 3419 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle) 3420 { 3421 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3422 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3423 union acpi_object *obj; 3424 acpi_status status; 3425 int ret; 3426 3427 if (!dev->driver) { 3428 /* dev->driver may be null if we're being removed */ 3429 dev_dbg(dev, "no driver found for dev\n"); 3430 return; 3431 } 3432 3433 if (!acpi_desc) { 3434 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3435 if (!acpi_desc) 3436 return; 3437 acpi_nfit_desc_init(acpi_desc, dev); 3438 } else { 3439 /* 3440 * Finish previous registration before considering new 3441 * regions. 3442 */ 3443 flush_workqueue(nfit_wq); 3444 } 3445 3446 /* Evaluate _FIT */ 3447 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf); 3448 if (ACPI_FAILURE(status)) { 3449 dev_err(dev, "failed to evaluate _FIT\n"); 3450 return; 3451 } 3452 3453 obj = buf.pointer; 3454 if (obj->type == ACPI_TYPE_BUFFER) { 3455 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3456 obj->buffer.length); 3457 if (ret) 3458 dev_err(dev, "failed to merge updated NFIT\n"); 3459 } else 3460 dev_err(dev, "Invalid _FIT\n"); 3461 kfree(buf.pointer); 3462 } 3463 3464 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle) 3465 { 3466 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3467 unsigned long flags = (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) ? 3468 0 : 1 << ARS_SHORT; 3469 3470 acpi_nfit_ars_rescan(acpi_desc, flags); 3471 } 3472 3473 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event) 3474 { 3475 dev_dbg(dev, "event: 0x%x\n", event); 3476 3477 switch (event) { 3478 case NFIT_NOTIFY_UPDATE: 3479 return acpi_nfit_update_notify(dev, handle); 3480 case NFIT_NOTIFY_UC_MEMORY_ERROR: 3481 return acpi_nfit_uc_error_notify(dev, handle); 3482 default: 3483 return; 3484 } 3485 } 3486 EXPORT_SYMBOL_GPL(__acpi_nfit_notify); 3487 3488 static void acpi_nfit_notify(struct acpi_device *adev, u32 event) 3489 { 3490 device_lock(&adev->dev); 3491 __acpi_nfit_notify(&adev->dev, adev->handle, event); 3492 device_unlock(&adev->dev); 3493 } 3494 3495 static const struct acpi_device_id acpi_nfit_ids[] = { 3496 { "ACPI0012", 0 }, 3497 { "", 0 }, 3498 }; 3499 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); 3500 3501 static struct acpi_driver acpi_nfit_driver = { 3502 .name = KBUILD_MODNAME, 3503 .ids = acpi_nfit_ids, 3504 .ops = { 3505 .add = acpi_nfit_add, 3506 .remove = acpi_nfit_remove, 3507 .notify = acpi_nfit_notify, 3508 }, 3509 }; 3510 3511 static __init int nfit_init(void) 3512 { 3513 int ret; 3514 3515 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); 3516 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); 3517 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); 3518 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); 3519 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); 3520 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); 3521 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); 3522 BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16); 3523 3524 guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]); 3525 guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]); 3526 guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]); 3527 guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]); 3528 guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]); 3529 guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]); 3530 guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]); 3531 guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]); 3532 guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]); 3533 guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]); 3534 guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); 3535 guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); 3536 guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); 3537 3538 nfit_wq = create_singlethread_workqueue("nfit"); 3539 if (!nfit_wq) 3540 return -ENOMEM; 3541 3542 nfit_mce_register(); 3543 ret = acpi_bus_register_driver(&acpi_nfit_driver); 3544 if (ret) { 3545 nfit_mce_unregister(); 3546 destroy_workqueue(nfit_wq); 3547 } 3548 3549 return ret; 3550 3551 } 3552 3553 static __exit void nfit_exit(void) 3554 { 3555 nfit_mce_unregister(); 3556 acpi_bus_unregister_driver(&acpi_nfit_driver); 3557 destroy_workqueue(nfit_wq); 3558 WARN_ON(!list_empty(&acpi_descs)); 3559 } 3560 3561 module_init(nfit_init); 3562 module_exit(nfit_exit); 3563 MODULE_LICENSE("GPL v2"); 3564 MODULE_AUTHOR("Intel Corporation"); 3565