1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 #include <linux/libnvdimm.h> 15 #include <linux/sched/mm.h> 16 #include <linux/vmalloc.h> 17 #include <linux/uaccess.h> 18 #include <linux/module.h> 19 #include <linux/blkdev.h> 20 #include <linux/fcntl.h> 21 #include <linux/async.h> 22 #include <linux/genhd.h> 23 #include <linux/ndctl.h> 24 #include <linux/sched.h> 25 #include <linux/slab.h> 26 #include <linux/fs.h> 27 #include <linux/io.h> 28 #include <linux/mm.h> 29 #include <linux/nd.h> 30 #include "nd-core.h" 31 #include "nd.h" 32 #include "pfn.h" 33 34 int nvdimm_major; 35 static int nvdimm_bus_major; 36 static struct class *nd_class; 37 static DEFINE_IDA(nd_ida); 38 39 static int to_nd_device_type(struct device *dev) 40 { 41 if (is_nvdimm(dev)) 42 return ND_DEVICE_DIMM; 43 else if (is_memory(dev)) 44 return ND_DEVICE_REGION_PMEM; 45 else if (is_nd_blk(dev)) 46 return ND_DEVICE_REGION_BLK; 47 else if (is_nd_dax(dev)) 48 return ND_DEVICE_DAX_PMEM; 49 else if (is_nd_region(dev->parent)) 50 return nd_region_to_nstype(to_nd_region(dev->parent)); 51 52 return 0; 53 } 54 55 static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env) 56 { 57 /* 58 * Ensure that region devices always have their numa node set as 59 * early as possible. 60 */ 61 if (is_nd_region(dev)) 62 set_dev_node(dev, to_nd_region(dev)->numa_node); 63 return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT, 64 to_nd_device_type(dev)); 65 } 66 67 static struct module *to_bus_provider(struct device *dev) 68 { 69 /* pin bus providers while regions are enabled */ 70 if (is_nd_region(dev)) { 71 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 72 73 return nvdimm_bus->nd_desc->module; 74 } 75 return NULL; 76 } 77 78 static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus) 79 { 80 nvdimm_bus_lock(&nvdimm_bus->dev); 81 nvdimm_bus->probe_active++; 82 nvdimm_bus_unlock(&nvdimm_bus->dev); 83 } 84 85 static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus) 86 { 87 nvdimm_bus_lock(&nvdimm_bus->dev); 88 if (--nvdimm_bus->probe_active == 0) 89 wake_up(&nvdimm_bus->probe_wait); 90 nvdimm_bus_unlock(&nvdimm_bus->dev); 91 } 92 93 static int nvdimm_bus_probe(struct device *dev) 94 { 95 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver); 96 struct module *provider = to_bus_provider(dev); 97 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 98 int rc; 99 100 if (!try_module_get(provider)) 101 return -ENXIO; 102 103 nvdimm_bus_probe_start(nvdimm_bus); 104 rc = nd_drv->probe(dev); 105 if (rc == 0) 106 nd_region_probe_success(nvdimm_bus, dev); 107 else 108 nd_region_disable(nvdimm_bus, dev); 109 nvdimm_bus_probe_end(nvdimm_bus); 110 111 dev_dbg(&nvdimm_bus->dev, "%s.probe(%s) = %d\n", dev->driver->name, 112 dev_name(dev), rc); 113 114 if (rc != 0) 115 module_put(provider); 116 return rc; 117 } 118 119 static int nvdimm_bus_remove(struct device *dev) 120 { 121 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver); 122 struct module *provider = to_bus_provider(dev); 123 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 124 int rc = 0; 125 126 if (nd_drv->remove) 127 rc = nd_drv->remove(dev); 128 nd_region_disable(nvdimm_bus, dev); 129 130 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name, 131 dev_name(dev), rc); 132 module_put(provider); 133 return rc; 134 } 135 136 static void nvdimm_bus_shutdown(struct device *dev) 137 { 138 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 139 struct nd_device_driver *nd_drv = NULL; 140 141 if (dev->driver) 142 nd_drv = to_nd_device_driver(dev->driver); 143 144 if (nd_drv && nd_drv->shutdown) { 145 nd_drv->shutdown(dev); 146 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n", 147 dev->driver->name, dev_name(dev)); 148 } 149 } 150 151 void nd_device_notify(struct device *dev, enum nvdimm_event event) 152 { 153 device_lock(dev); 154 if (dev->driver) { 155 struct nd_device_driver *nd_drv; 156 157 nd_drv = to_nd_device_driver(dev->driver); 158 if (nd_drv->notify) 159 nd_drv->notify(dev, event); 160 } 161 device_unlock(dev); 162 } 163 EXPORT_SYMBOL(nd_device_notify); 164 165 void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event) 166 { 167 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev); 168 169 if (!nvdimm_bus) 170 return; 171 172 /* caller is responsible for holding a reference on the device */ 173 nd_device_notify(&nd_region->dev, event); 174 } 175 EXPORT_SYMBOL_GPL(nvdimm_region_notify); 176 177 struct clear_badblocks_context { 178 resource_size_t phys, cleared; 179 }; 180 181 static int nvdimm_clear_badblocks_region(struct device *dev, void *data) 182 { 183 struct clear_badblocks_context *ctx = data; 184 struct nd_region *nd_region; 185 resource_size_t ndr_end; 186 sector_t sector; 187 188 /* make sure device is a region */ 189 if (!is_nd_pmem(dev)) 190 return 0; 191 192 nd_region = to_nd_region(dev); 193 ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1; 194 195 /* make sure we are in the region */ 196 if (ctx->phys < nd_region->ndr_start 197 || (ctx->phys + ctx->cleared) > ndr_end) 198 return 0; 199 200 sector = (ctx->phys - nd_region->ndr_start) / 512; 201 badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512); 202 203 if (nd_region->bb_state) 204 sysfs_notify_dirent(nd_region->bb_state); 205 206 return 0; 207 } 208 209 static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus, 210 phys_addr_t phys, u64 cleared) 211 { 212 struct clear_badblocks_context ctx = { 213 .phys = phys, 214 .cleared = cleared, 215 }; 216 217 device_for_each_child(&nvdimm_bus->dev, &ctx, 218 nvdimm_clear_badblocks_region); 219 } 220 221 static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus, 222 phys_addr_t phys, u64 cleared) 223 { 224 if (cleared > 0) 225 badrange_forget(&nvdimm_bus->badrange, phys, cleared); 226 227 if (cleared > 0 && cleared / 512) 228 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared); 229 } 230 231 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys, 232 unsigned int len) 233 { 234 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 235 struct nvdimm_bus_descriptor *nd_desc; 236 struct nd_cmd_clear_error clear_err; 237 struct nd_cmd_ars_cap ars_cap; 238 u32 clear_err_unit, mask; 239 unsigned int noio_flag; 240 int cmd_rc, rc; 241 242 if (!nvdimm_bus) 243 return -ENXIO; 244 245 nd_desc = nvdimm_bus->nd_desc; 246 /* 247 * if ndctl does not exist, it's PMEM_LEGACY and 248 * we want to just pretend everything is handled. 249 */ 250 if (!nd_desc->ndctl) 251 return len; 252 253 memset(&ars_cap, 0, sizeof(ars_cap)); 254 ars_cap.address = phys; 255 ars_cap.length = len; 256 noio_flag = memalloc_noio_save(); 257 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap, 258 sizeof(ars_cap), &cmd_rc); 259 memalloc_noio_restore(noio_flag); 260 if (rc < 0) 261 return rc; 262 if (cmd_rc < 0) 263 return cmd_rc; 264 clear_err_unit = ars_cap.clear_err_unit; 265 if (!clear_err_unit || !is_power_of_2(clear_err_unit)) 266 return -ENXIO; 267 268 mask = clear_err_unit - 1; 269 if ((phys | len) & mask) 270 return -ENXIO; 271 memset(&clear_err, 0, sizeof(clear_err)); 272 clear_err.address = phys; 273 clear_err.length = len; 274 noio_flag = memalloc_noio_save(); 275 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err, 276 sizeof(clear_err), &cmd_rc); 277 memalloc_noio_restore(noio_flag); 278 if (rc < 0) 279 return rc; 280 if (cmd_rc < 0) 281 return cmd_rc; 282 283 nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared); 284 285 return clear_err.cleared; 286 } 287 EXPORT_SYMBOL_GPL(nvdimm_clear_poison); 288 289 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv); 290 291 static struct bus_type nvdimm_bus_type = { 292 .name = "nd", 293 .uevent = nvdimm_bus_uevent, 294 .match = nvdimm_bus_match, 295 .probe = nvdimm_bus_probe, 296 .remove = nvdimm_bus_remove, 297 .shutdown = nvdimm_bus_shutdown, 298 }; 299 300 static void nvdimm_bus_release(struct device *dev) 301 { 302 struct nvdimm_bus *nvdimm_bus; 303 304 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); 305 ida_simple_remove(&nd_ida, nvdimm_bus->id); 306 kfree(nvdimm_bus); 307 } 308 309 static bool is_nvdimm_bus(struct device *dev) 310 { 311 return dev->release == nvdimm_bus_release; 312 } 313 314 struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev) 315 { 316 struct device *dev; 317 318 for (dev = nd_dev; dev; dev = dev->parent) 319 if (is_nvdimm_bus(dev)) 320 break; 321 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n"); 322 if (dev) 323 return to_nvdimm_bus(dev); 324 return NULL; 325 } 326 327 struct nvdimm_bus *to_nvdimm_bus(struct device *dev) 328 { 329 struct nvdimm_bus *nvdimm_bus; 330 331 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); 332 WARN_ON(!is_nvdimm_bus(dev)); 333 return nvdimm_bus; 334 } 335 EXPORT_SYMBOL_GPL(to_nvdimm_bus); 336 337 struct nvdimm_bus *nvdimm_bus_register(struct device *parent, 338 struct nvdimm_bus_descriptor *nd_desc) 339 { 340 struct nvdimm_bus *nvdimm_bus; 341 int rc; 342 343 nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL); 344 if (!nvdimm_bus) 345 return NULL; 346 INIT_LIST_HEAD(&nvdimm_bus->list); 347 INIT_LIST_HEAD(&nvdimm_bus->mapping_list); 348 init_waitqueue_head(&nvdimm_bus->probe_wait); 349 nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL); 350 mutex_init(&nvdimm_bus->reconfig_mutex); 351 badrange_init(&nvdimm_bus->badrange); 352 if (nvdimm_bus->id < 0) { 353 kfree(nvdimm_bus); 354 return NULL; 355 } 356 nvdimm_bus->nd_desc = nd_desc; 357 nvdimm_bus->dev.parent = parent; 358 nvdimm_bus->dev.release = nvdimm_bus_release; 359 nvdimm_bus->dev.groups = nd_desc->attr_groups; 360 nvdimm_bus->dev.bus = &nvdimm_bus_type; 361 dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id); 362 rc = device_register(&nvdimm_bus->dev); 363 if (rc) { 364 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc); 365 goto err; 366 } 367 368 return nvdimm_bus; 369 err: 370 put_device(&nvdimm_bus->dev); 371 return NULL; 372 } 373 EXPORT_SYMBOL_GPL(nvdimm_bus_register); 374 375 void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus) 376 { 377 if (!nvdimm_bus) 378 return; 379 device_unregister(&nvdimm_bus->dev); 380 } 381 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister); 382 383 static int child_unregister(struct device *dev, void *data) 384 { 385 /* 386 * the singular ndctl class device per bus needs to be 387 * "device_destroy"ed, so skip it here 388 * 389 * i.e. remove classless children 390 */ 391 if (dev->class) 392 /* pass */; 393 else 394 nd_device_unregister(dev, ND_SYNC); 395 return 0; 396 } 397 398 static void free_badrange_list(struct list_head *badrange_list) 399 { 400 struct badrange_entry *bre, *next; 401 402 list_for_each_entry_safe(bre, next, badrange_list, list) { 403 list_del(&bre->list); 404 kfree(bre); 405 } 406 list_del_init(badrange_list); 407 } 408 409 static int nd_bus_remove(struct device *dev) 410 { 411 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 412 413 mutex_lock(&nvdimm_bus_list_mutex); 414 list_del_init(&nvdimm_bus->list); 415 mutex_unlock(&nvdimm_bus_list_mutex); 416 417 nd_synchronize(); 418 device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister); 419 420 spin_lock(&nvdimm_bus->badrange.lock); 421 free_badrange_list(&nvdimm_bus->badrange.list); 422 spin_unlock(&nvdimm_bus->badrange.lock); 423 424 nvdimm_bus_destroy_ndctl(nvdimm_bus); 425 426 return 0; 427 } 428 429 static int nd_bus_probe(struct device *dev) 430 { 431 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 432 int rc; 433 434 rc = nvdimm_bus_create_ndctl(nvdimm_bus); 435 if (rc) 436 return rc; 437 438 mutex_lock(&nvdimm_bus_list_mutex); 439 list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list); 440 mutex_unlock(&nvdimm_bus_list_mutex); 441 442 /* enable bus provider attributes to look up their local context */ 443 dev_set_drvdata(dev, nvdimm_bus->nd_desc); 444 445 return 0; 446 } 447 448 static struct nd_device_driver nd_bus_driver = { 449 .probe = nd_bus_probe, 450 .remove = nd_bus_remove, 451 .drv = { 452 .name = "nd_bus", 453 .suppress_bind_attrs = true, 454 .bus = &nvdimm_bus_type, 455 .owner = THIS_MODULE, 456 .mod_name = KBUILD_MODNAME, 457 }, 458 }; 459 460 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv) 461 { 462 struct nd_device_driver *nd_drv = to_nd_device_driver(drv); 463 464 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver) 465 return true; 466 467 return !!test_bit(to_nd_device_type(dev), &nd_drv->type); 468 } 469 470 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain); 471 472 void nd_synchronize(void) 473 { 474 async_synchronize_full_domain(&nd_async_domain); 475 } 476 EXPORT_SYMBOL_GPL(nd_synchronize); 477 478 static void nd_async_device_register(void *d, async_cookie_t cookie) 479 { 480 struct device *dev = d; 481 482 if (device_add(dev) != 0) { 483 dev_err(dev, "%s: failed\n", __func__); 484 put_device(dev); 485 } 486 put_device(dev); 487 } 488 489 static void nd_async_device_unregister(void *d, async_cookie_t cookie) 490 { 491 struct device *dev = d; 492 493 /* flush bus operations before delete */ 494 nvdimm_bus_lock(dev); 495 nvdimm_bus_unlock(dev); 496 497 device_unregister(dev); 498 put_device(dev); 499 } 500 501 void __nd_device_register(struct device *dev) 502 { 503 if (!dev) 504 return; 505 dev->bus = &nvdimm_bus_type; 506 get_device(dev); 507 async_schedule_domain(nd_async_device_register, dev, 508 &nd_async_domain); 509 } 510 511 void nd_device_register(struct device *dev) 512 { 513 device_initialize(dev); 514 __nd_device_register(dev); 515 } 516 EXPORT_SYMBOL(nd_device_register); 517 518 void nd_device_unregister(struct device *dev, enum nd_async_mode mode) 519 { 520 switch (mode) { 521 case ND_ASYNC: 522 get_device(dev); 523 async_schedule_domain(nd_async_device_unregister, dev, 524 &nd_async_domain); 525 break; 526 case ND_SYNC: 527 nd_synchronize(); 528 device_unregister(dev); 529 break; 530 } 531 } 532 EXPORT_SYMBOL(nd_device_unregister); 533 534 /** 535 * __nd_driver_register() - register a region or a namespace driver 536 * @nd_drv: driver to register 537 * @owner: automatically set by nd_driver_register() macro 538 * @mod_name: automatically set by nd_driver_register() macro 539 */ 540 int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner, 541 const char *mod_name) 542 { 543 struct device_driver *drv = &nd_drv->drv; 544 545 if (!nd_drv->type) { 546 pr_debug("driver type bitmask not set (%pf)\n", 547 __builtin_return_address(0)); 548 return -EINVAL; 549 } 550 551 if (!nd_drv->probe) { 552 pr_debug("%s ->probe() must be specified\n", mod_name); 553 return -EINVAL; 554 } 555 556 drv->bus = &nvdimm_bus_type; 557 drv->owner = owner; 558 drv->mod_name = mod_name; 559 560 return driver_register(drv); 561 } 562 EXPORT_SYMBOL(__nd_driver_register); 563 564 int nvdimm_revalidate_disk(struct gendisk *disk) 565 { 566 struct device *dev = disk_to_dev(disk)->parent; 567 struct nd_region *nd_region = to_nd_region(dev->parent); 568 const char *pol = nd_region->ro ? "only" : "write"; 569 570 if (nd_region->ro == get_disk_ro(disk)) 571 return 0; 572 573 dev_info(dev, "%s read-%s, marking %s read-%s\n", 574 dev_name(&nd_region->dev), pol, disk->disk_name, pol); 575 set_disk_ro(disk, nd_region->ro); 576 577 return 0; 578 579 } 580 EXPORT_SYMBOL(nvdimm_revalidate_disk); 581 582 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, 583 char *buf) 584 { 585 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n", 586 to_nd_device_type(dev)); 587 } 588 static DEVICE_ATTR_RO(modalias); 589 590 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr, 591 char *buf) 592 { 593 return sprintf(buf, "%s\n", dev->type->name); 594 } 595 static DEVICE_ATTR_RO(devtype); 596 597 static struct attribute *nd_device_attributes[] = { 598 &dev_attr_modalias.attr, 599 &dev_attr_devtype.attr, 600 NULL, 601 }; 602 603 /** 604 * nd_device_attribute_group - generic attributes for all devices on an nd bus 605 */ 606 struct attribute_group nd_device_attribute_group = { 607 .attrs = nd_device_attributes, 608 }; 609 EXPORT_SYMBOL_GPL(nd_device_attribute_group); 610 611 static ssize_t numa_node_show(struct device *dev, 612 struct device_attribute *attr, char *buf) 613 { 614 return sprintf(buf, "%d\n", dev_to_node(dev)); 615 } 616 static DEVICE_ATTR_RO(numa_node); 617 618 static struct attribute *nd_numa_attributes[] = { 619 &dev_attr_numa_node.attr, 620 NULL, 621 }; 622 623 static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a, 624 int n) 625 { 626 if (!IS_ENABLED(CONFIG_NUMA)) 627 return 0; 628 629 return a->mode; 630 } 631 632 /** 633 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus 634 */ 635 struct attribute_group nd_numa_attribute_group = { 636 .attrs = nd_numa_attributes, 637 .is_visible = nd_numa_attr_visible, 638 }; 639 EXPORT_SYMBOL_GPL(nd_numa_attribute_group); 640 641 int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus) 642 { 643 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id); 644 struct device *dev; 645 646 dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus, 647 "ndctl%d", nvdimm_bus->id); 648 649 if (IS_ERR(dev)) 650 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n", 651 nvdimm_bus->id, PTR_ERR(dev)); 652 return PTR_ERR_OR_ZERO(dev); 653 } 654 655 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus) 656 { 657 device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id)); 658 } 659 660 static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = { 661 [ND_CMD_IMPLEMENTED] = { }, 662 [ND_CMD_SMART] = { 663 .out_num = 2, 664 .out_sizes = { 4, 128, }, 665 }, 666 [ND_CMD_SMART_THRESHOLD] = { 667 .out_num = 2, 668 .out_sizes = { 4, 8, }, 669 }, 670 [ND_CMD_DIMM_FLAGS] = { 671 .out_num = 2, 672 .out_sizes = { 4, 4 }, 673 }, 674 [ND_CMD_GET_CONFIG_SIZE] = { 675 .out_num = 3, 676 .out_sizes = { 4, 4, 4, }, 677 }, 678 [ND_CMD_GET_CONFIG_DATA] = { 679 .in_num = 2, 680 .in_sizes = { 4, 4, }, 681 .out_num = 2, 682 .out_sizes = { 4, UINT_MAX, }, 683 }, 684 [ND_CMD_SET_CONFIG_DATA] = { 685 .in_num = 3, 686 .in_sizes = { 4, 4, UINT_MAX, }, 687 .out_num = 1, 688 .out_sizes = { 4, }, 689 }, 690 [ND_CMD_VENDOR] = { 691 .in_num = 3, 692 .in_sizes = { 4, 4, UINT_MAX, }, 693 .out_num = 3, 694 .out_sizes = { 4, 4, UINT_MAX, }, 695 }, 696 [ND_CMD_CALL] = { 697 .in_num = 2, 698 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, }, 699 .out_num = 1, 700 .out_sizes = { UINT_MAX, }, 701 }, 702 }; 703 704 const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd) 705 { 706 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs)) 707 return &__nd_cmd_dimm_descs[cmd]; 708 return NULL; 709 } 710 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc); 711 712 static const struct nd_cmd_desc __nd_cmd_bus_descs[] = { 713 [ND_CMD_IMPLEMENTED] = { }, 714 [ND_CMD_ARS_CAP] = { 715 .in_num = 2, 716 .in_sizes = { 8, 8, }, 717 .out_num = 4, 718 .out_sizes = { 4, 4, 4, 4, }, 719 }, 720 [ND_CMD_ARS_START] = { 721 .in_num = 5, 722 .in_sizes = { 8, 8, 2, 1, 5, }, 723 .out_num = 2, 724 .out_sizes = { 4, 4, }, 725 }, 726 [ND_CMD_ARS_STATUS] = { 727 .out_num = 3, 728 .out_sizes = { 4, 4, UINT_MAX, }, 729 }, 730 [ND_CMD_CLEAR_ERROR] = { 731 .in_num = 2, 732 .in_sizes = { 8, 8, }, 733 .out_num = 3, 734 .out_sizes = { 4, 4, 8, }, 735 }, 736 [ND_CMD_CALL] = { 737 .in_num = 2, 738 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, }, 739 .out_num = 1, 740 .out_sizes = { UINT_MAX, }, 741 }, 742 }; 743 744 const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd) 745 { 746 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs)) 747 return &__nd_cmd_bus_descs[cmd]; 748 return NULL; 749 } 750 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc); 751 752 u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd, 753 const struct nd_cmd_desc *desc, int idx, void *buf) 754 { 755 if (idx >= desc->in_num) 756 return UINT_MAX; 757 758 if (desc->in_sizes[idx] < UINT_MAX) 759 return desc->in_sizes[idx]; 760 761 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) { 762 struct nd_cmd_set_config_hdr *hdr = buf; 763 764 return hdr->in_length; 765 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) { 766 struct nd_cmd_vendor_hdr *hdr = buf; 767 768 return hdr->in_length; 769 } else if (cmd == ND_CMD_CALL) { 770 struct nd_cmd_pkg *pkg = buf; 771 772 return pkg->nd_size_in; 773 } 774 775 return UINT_MAX; 776 } 777 EXPORT_SYMBOL_GPL(nd_cmd_in_size); 778 779 u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd, 780 const struct nd_cmd_desc *desc, int idx, const u32 *in_field, 781 const u32 *out_field, unsigned long remainder) 782 { 783 if (idx >= desc->out_num) 784 return UINT_MAX; 785 786 if (desc->out_sizes[idx] < UINT_MAX) 787 return desc->out_sizes[idx]; 788 789 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1) 790 return in_field[1]; 791 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) 792 return out_field[1]; 793 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) { 794 /* 795 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is 796 * "Size of Output Buffer in bytes, including this 797 * field." 798 */ 799 if (out_field[1] < 4) 800 return 0; 801 /* 802 * ACPI 6.1 is ambiguous if 'status' is included in the 803 * output size. If we encounter an output size that 804 * overshoots the remainder by 4 bytes, assume it was 805 * including 'status'. 806 */ 807 if (out_field[1] - 8 == remainder) 808 return remainder; 809 return out_field[1] - 4; 810 } else if (cmd == ND_CMD_CALL) { 811 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field; 812 813 return pkg->nd_size_out; 814 } 815 816 817 return UINT_MAX; 818 } 819 EXPORT_SYMBOL_GPL(nd_cmd_out_size); 820 821 void wait_nvdimm_bus_probe_idle(struct device *dev) 822 { 823 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 824 825 do { 826 if (nvdimm_bus->probe_active == 0) 827 break; 828 nvdimm_bus_unlock(&nvdimm_bus->dev); 829 wait_event(nvdimm_bus->probe_wait, 830 nvdimm_bus->probe_active == 0); 831 nvdimm_bus_lock(&nvdimm_bus->dev); 832 } while (true); 833 } 834 835 static int nd_pmem_forget_poison_check(struct device *dev, void *data) 836 { 837 struct nd_cmd_clear_error *clear_err = 838 (struct nd_cmd_clear_error *)data; 839 struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL; 840 struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL; 841 struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL; 842 struct nd_namespace_common *ndns = NULL; 843 struct nd_namespace_io *nsio; 844 resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend; 845 846 if (nd_dax || !dev->driver) 847 return 0; 848 849 start = clear_err->address; 850 end = clear_err->address + clear_err->cleared - 1; 851 852 if (nd_btt || nd_pfn || nd_dax) { 853 if (nd_btt) 854 ndns = nd_btt->ndns; 855 else if (nd_pfn) 856 ndns = nd_pfn->ndns; 857 else if (nd_dax) 858 ndns = nd_dax->nd_pfn.ndns; 859 860 if (!ndns) 861 return 0; 862 } else 863 ndns = to_ndns(dev); 864 865 nsio = to_nd_namespace_io(&ndns->dev); 866 pstart = nsio->res.start + offset; 867 pend = nsio->res.end - end_trunc; 868 869 if ((pstart >= start) && (pend <= end)) 870 return -EBUSY; 871 872 return 0; 873 874 } 875 876 static int nd_ns_forget_poison_check(struct device *dev, void *data) 877 { 878 return device_for_each_child(dev, data, nd_pmem_forget_poison_check); 879 } 880 881 /* set_config requires an idle interleave set */ 882 static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus, 883 struct nvdimm *nvdimm, unsigned int cmd, void *data) 884 { 885 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc; 886 887 /* ask the bus provider if it would like to block this request */ 888 if (nd_desc->clear_to_send) { 889 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd); 890 891 if (rc) 892 return rc; 893 } 894 895 /* require clear error to go through the pmem driver */ 896 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR) 897 return device_for_each_child(&nvdimm_bus->dev, data, 898 nd_ns_forget_poison_check); 899 900 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA) 901 return 0; 902 903 /* prevent label manipulation while the kernel owns label updates */ 904 wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev); 905 if (atomic_read(&nvdimm->busy)) 906 return -EBUSY; 907 return 0; 908 } 909 910 static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm, 911 int read_only, unsigned int ioctl_cmd, unsigned long arg) 912 { 913 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc; 914 static char out_env[ND_CMD_MAX_ENVELOPE]; 915 static char in_env[ND_CMD_MAX_ENVELOPE]; 916 const struct nd_cmd_desc *desc = NULL; 917 unsigned int cmd = _IOC_NR(ioctl_cmd); 918 struct device *dev = &nvdimm_bus->dev; 919 void __user *p = (void __user *) arg; 920 const char *cmd_name, *dimm_name; 921 u32 in_len = 0, out_len = 0; 922 unsigned int func = cmd; 923 unsigned long cmd_mask; 924 struct nd_cmd_pkg pkg; 925 int rc, i, cmd_rc; 926 u64 buf_len = 0; 927 void *buf; 928 929 if (nvdimm) { 930 desc = nd_cmd_dimm_desc(cmd); 931 cmd_name = nvdimm_cmd_name(cmd); 932 cmd_mask = nvdimm->cmd_mask; 933 dimm_name = dev_name(&nvdimm->dev); 934 } else { 935 desc = nd_cmd_bus_desc(cmd); 936 cmd_name = nvdimm_bus_cmd_name(cmd); 937 cmd_mask = nd_desc->cmd_mask; 938 dimm_name = "bus"; 939 } 940 941 if (cmd == ND_CMD_CALL) { 942 if (copy_from_user(&pkg, p, sizeof(pkg))) 943 return -EFAULT; 944 } 945 946 if (!desc || (desc->out_num + desc->in_num == 0) || 947 !test_bit(cmd, &cmd_mask)) 948 return -ENOTTY; 949 950 /* fail write commands (when read-only) */ 951 if (read_only) 952 switch (cmd) { 953 case ND_CMD_VENDOR: 954 case ND_CMD_SET_CONFIG_DATA: 955 case ND_CMD_ARS_START: 956 case ND_CMD_CLEAR_ERROR: 957 case ND_CMD_CALL: 958 dev_dbg(&nvdimm_bus->dev, "'%s' command while read-only.\n", 959 nvdimm ? nvdimm_cmd_name(cmd) 960 : nvdimm_bus_cmd_name(cmd)); 961 return -EPERM; 962 default: 963 break; 964 } 965 966 /* process an input envelope */ 967 for (i = 0; i < desc->in_num; i++) { 968 u32 in_size, copy; 969 970 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env); 971 if (in_size == UINT_MAX) { 972 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n", 973 __func__, dimm_name, cmd_name, i); 974 return -ENXIO; 975 } 976 if (in_len < sizeof(in_env)) 977 copy = min_t(u32, sizeof(in_env) - in_len, in_size); 978 else 979 copy = 0; 980 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) 981 return -EFAULT; 982 in_len += in_size; 983 } 984 985 if (cmd == ND_CMD_CALL) { 986 func = pkg.nd_command; 987 dev_dbg(dev, "%s:%s, idx: %llu, in: %u, out: %u, len %llu\n", 988 __func__, dimm_name, pkg.nd_command, 989 in_len, out_len, buf_len); 990 } 991 992 /* process an output envelope */ 993 for (i = 0; i < desc->out_num; i++) { 994 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, 995 (u32 *) in_env, (u32 *) out_env, 0); 996 u32 copy; 997 998 if (out_size == UINT_MAX) { 999 dev_dbg(dev, "%s:%s unknown output size cmd: %s field: %d\n", 1000 __func__, dimm_name, cmd_name, i); 1001 return -EFAULT; 1002 } 1003 if (out_len < sizeof(out_env)) 1004 copy = min_t(u32, sizeof(out_env) - out_len, out_size); 1005 else 1006 copy = 0; 1007 if (copy && copy_from_user(&out_env[out_len], 1008 p + in_len + out_len, copy)) 1009 return -EFAULT; 1010 out_len += out_size; 1011 } 1012 1013 buf_len = (u64) out_len + (u64) in_len; 1014 if (buf_len > ND_IOCTL_MAX_BUFLEN) { 1015 dev_dbg(dev, "%s:%s cmd: %s buf_len: %llu > %d\n", __func__, 1016 dimm_name, cmd_name, buf_len, 1017 ND_IOCTL_MAX_BUFLEN); 1018 return -EINVAL; 1019 } 1020 1021 buf = vmalloc(buf_len); 1022 if (!buf) 1023 return -ENOMEM; 1024 1025 if (copy_from_user(buf, p, buf_len)) { 1026 rc = -EFAULT; 1027 goto out; 1028 } 1029 1030 nvdimm_bus_lock(&nvdimm_bus->dev); 1031 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf); 1032 if (rc) 1033 goto out_unlock; 1034 1035 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc); 1036 if (rc < 0) 1037 goto out_unlock; 1038 1039 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) { 1040 struct nd_cmd_clear_error *clear_err = buf; 1041 1042 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address, 1043 clear_err->cleared); 1044 } 1045 nvdimm_bus_unlock(&nvdimm_bus->dev); 1046 1047 if (copy_to_user(p, buf, buf_len)) 1048 rc = -EFAULT; 1049 1050 vfree(buf); 1051 return rc; 1052 1053 out_unlock: 1054 nvdimm_bus_unlock(&nvdimm_bus->dev); 1055 out: 1056 vfree(buf); 1057 return rc; 1058 } 1059 1060 static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 1061 { 1062 long id = (long) file->private_data; 1063 int rc = -ENXIO, ro; 1064 struct nvdimm_bus *nvdimm_bus; 1065 1066 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY); 1067 mutex_lock(&nvdimm_bus_list_mutex); 1068 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) { 1069 if (nvdimm_bus->id == id) { 1070 rc = __nd_ioctl(nvdimm_bus, NULL, ro, cmd, arg); 1071 break; 1072 } 1073 } 1074 mutex_unlock(&nvdimm_bus_list_mutex); 1075 1076 return rc; 1077 } 1078 1079 static int match_dimm(struct device *dev, void *data) 1080 { 1081 long id = (long) data; 1082 1083 if (is_nvdimm(dev)) { 1084 struct nvdimm *nvdimm = to_nvdimm(dev); 1085 1086 return nvdimm->id == id; 1087 } 1088 1089 return 0; 1090 } 1091 1092 static long nvdimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 1093 { 1094 int rc = -ENXIO, ro; 1095 struct nvdimm_bus *nvdimm_bus; 1096 1097 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY); 1098 mutex_lock(&nvdimm_bus_list_mutex); 1099 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) { 1100 struct device *dev = device_find_child(&nvdimm_bus->dev, 1101 file->private_data, match_dimm); 1102 struct nvdimm *nvdimm; 1103 1104 if (!dev) 1105 continue; 1106 1107 nvdimm = to_nvdimm(dev); 1108 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg); 1109 put_device(dev); 1110 break; 1111 } 1112 mutex_unlock(&nvdimm_bus_list_mutex); 1113 1114 return rc; 1115 } 1116 1117 static int nd_open(struct inode *inode, struct file *file) 1118 { 1119 long minor = iminor(inode); 1120 1121 file->private_data = (void *) minor; 1122 return 0; 1123 } 1124 1125 static const struct file_operations nvdimm_bus_fops = { 1126 .owner = THIS_MODULE, 1127 .open = nd_open, 1128 .unlocked_ioctl = nd_ioctl, 1129 .compat_ioctl = nd_ioctl, 1130 .llseek = noop_llseek, 1131 }; 1132 1133 static const struct file_operations nvdimm_fops = { 1134 .owner = THIS_MODULE, 1135 .open = nd_open, 1136 .unlocked_ioctl = nvdimm_ioctl, 1137 .compat_ioctl = nvdimm_ioctl, 1138 .llseek = noop_llseek, 1139 }; 1140 1141 int __init nvdimm_bus_init(void) 1142 { 1143 int rc; 1144 1145 rc = bus_register(&nvdimm_bus_type); 1146 if (rc) 1147 return rc; 1148 1149 rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops); 1150 if (rc < 0) 1151 goto err_bus_chrdev; 1152 nvdimm_bus_major = rc; 1153 1154 rc = register_chrdev(0, "dimmctl", &nvdimm_fops); 1155 if (rc < 0) 1156 goto err_dimm_chrdev; 1157 nvdimm_major = rc; 1158 1159 nd_class = class_create(THIS_MODULE, "nd"); 1160 if (IS_ERR(nd_class)) { 1161 rc = PTR_ERR(nd_class); 1162 goto err_class; 1163 } 1164 1165 rc = driver_register(&nd_bus_driver.drv); 1166 if (rc) 1167 goto err_nd_bus; 1168 1169 return 0; 1170 1171 err_nd_bus: 1172 class_destroy(nd_class); 1173 err_class: 1174 unregister_chrdev(nvdimm_major, "dimmctl"); 1175 err_dimm_chrdev: 1176 unregister_chrdev(nvdimm_bus_major, "ndctl"); 1177 err_bus_chrdev: 1178 bus_unregister(&nvdimm_bus_type); 1179 1180 return rc; 1181 } 1182 1183 void nvdimm_bus_exit(void) 1184 { 1185 driver_unregister(&nd_bus_driver.drv); 1186 class_destroy(nd_class); 1187 unregister_chrdev(nvdimm_bus_major, "ndctl"); 1188 unregister_chrdev(nvdimm_major, "dimmctl"); 1189 bus_unregister(&nvdimm_bus_type); 1190 ida_destroy(&nd_ida); 1191 } 1192