1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 #include <linux/vmalloc.h> 15 #include <linux/uaccess.h> 16 #include <linux/module.h> 17 #include <linux/blkdev.h> 18 #include <linux/fcntl.h> 19 #include <linux/async.h> 20 #include <linux/genhd.h> 21 #include <linux/ndctl.h> 22 #include <linux/sched.h> 23 #include <linux/slab.h> 24 #include <linux/fs.h> 25 #include <linux/io.h> 26 #include <linux/mm.h> 27 #include <linux/nd.h> 28 #include "nd-core.h" 29 #include "nd.h" 30 31 int nvdimm_major; 32 static int nvdimm_bus_major; 33 static struct class *nd_class; 34 static DEFINE_IDA(nd_ida); 35 36 static int to_nd_device_type(struct device *dev) 37 { 38 if (is_nvdimm(dev)) 39 return ND_DEVICE_DIMM; 40 else if (is_nd_pmem(dev)) 41 return ND_DEVICE_REGION_PMEM; 42 else if (is_nd_blk(dev)) 43 return ND_DEVICE_REGION_BLK; 44 else if (is_nd_dax(dev)) 45 return ND_DEVICE_DAX_PMEM; 46 else if (is_nd_pmem(dev->parent) || is_nd_blk(dev->parent)) 47 return nd_region_to_nstype(to_nd_region(dev->parent)); 48 49 return 0; 50 } 51 52 static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env) 53 { 54 /* 55 * Ensure that region devices always have their numa node set as 56 * early as possible. 57 */ 58 if (is_nd_pmem(dev) || is_nd_blk(dev)) 59 set_dev_node(dev, to_nd_region(dev)->numa_node); 60 return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT, 61 to_nd_device_type(dev)); 62 } 63 64 static struct module *to_bus_provider(struct device *dev) 65 { 66 /* pin bus providers while regions are enabled */ 67 if (is_nd_pmem(dev) || is_nd_blk(dev)) { 68 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 69 70 return nvdimm_bus->nd_desc->module; 71 } 72 return NULL; 73 } 74 75 static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus) 76 { 77 nvdimm_bus_lock(&nvdimm_bus->dev); 78 nvdimm_bus->probe_active++; 79 nvdimm_bus_unlock(&nvdimm_bus->dev); 80 } 81 82 static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus) 83 { 84 nvdimm_bus_lock(&nvdimm_bus->dev); 85 if (--nvdimm_bus->probe_active == 0) 86 wake_up(&nvdimm_bus->probe_wait); 87 nvdimm_bus_unlock(&nvdimm_bus->dev); 88 } 89 90 static int nvdimm_bus_probe(struct device *dev) 91 { 92 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver); 93 struct module *provider = to_bus_provider(dev); 94 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 95 int rc; 96 97 if (!try_module_get(provider)) 98 return -ENXIO; 99 100 nvdimm_bus_probe_start(nvdimm_bus); 101 rc = nd_drv->probe(dev); 102 if (rc == 0) 103 nd_region_probe_success(nvdimm_bus, dev); 104 else 105 nd_region_disable(nvdimm_bus, dev); 106 nvdimm_bus_probe_end(nvdimm_bus); 107 108 dev_dbg(&nvdimm_bus->dev, "%s.probe(%s) = %d\n", dev->driver->name, 109 dev_name(dev), rc); 110 111 if (rc != 0) 112 module_put(provider); 113 return rc; 114 } 115 116 static int nvdimm_bus_remove(struct device *dev) 117 { 118 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver); 119 struct module *provider = to_bus_provider(dev); 120 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 121 int rc = 0; 122 123 if (nd_drv->remove) 124 rc = nd_drv->remove(dev); 125 nd_region_disable(nvdimm_bus, dev); 126 127 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name, 128 dev_name(dev), rc); 129 module_put(provider); 130 return rc; 131 } 132 133 static void nvdimm_bus_shutdown(struct device *dev) 134 { 135 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 136 struct nd_device_driver *nd_drv = NULL; 137 138 if (dev->driver) 139 nd_drv = to_nd_device_driver(dev->driver); 140 141 if (nd_drv && nd_drv->shutdown) { 142 nd_drv->shutdown(dev); 143 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n", 144 dev->driver->name, dev_name(dev)); 145 } 146 } 147 148 void nd_device_notify(struct device *dev, enum nvdimm_event event) 149 { 150 device_lock(dev); 151 if (dev->driver) { 152 struct nd_device_driver *nd_drv; 153 154 nd_drv = to_nd_device_driver(dev->driver); 155 if (nd_drv->notify) 156 nd_drv->notify(dev, event); 157 } 158 device_unlock(dev); 159 } 160 EXPORT_SYMBOL(nd_device_notify); 161 162 void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event) 163 { 164 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev); 165 166 if (!nvdimm_bus) 167 return; 168 169 /* caller is responsible for holding a reference on the device */ 170 nd_device_notify(&nd_region->dev, event); 171 } 172 EXPORT_SYMBOL_GPL(nvdimm_region_notify); 173 174 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys, 175 unsigned int len) 176 { 177 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 178 struct nvdimm_bus_descriptor *nd_desc; 179 struct nd_cmd_clear_error clear_err; 180 struct nd_cmd_ars_cap ars_cap; 181 u32 clear_err_unit, mask; 182 int cmd_rc, rc; 183 184 if (!nvdimm_bus) 185 return -ENXIO; 186 187 nd_desc = nvdimm_bus->nd_desc; 188 /* 189 * if ndctl does not exist, it's PMEM_LEGACY and 190 * we want to just pretend everything is handled. 191 */ 192 if (!nd_desc->ndctl) 193 return len; 194 195 memset(&ars_cap, 0, sizeof(ars_cap)); 196 ars_cap.address = phys; 197 ars_cap.length = len; 198 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap, 199 sizeof(ars_cap), &cmd_rc); 200 if (rc < 0) 201 return rc; 202 if (cmd_rc < 0) 203 return cmd_rc; 204 clear_err_unit = ars_cap.clear_err_unit; 205 if (!clear_err_unit || !is_power_of_2(clear_err_unit)) 206 return -ENXIO; 207 208 mask = clear_err_unit - 1; 209 if ((phys | len) & mask) 210 return -ENXIO; 211 memset(&clear_err, 0, sizeof(clear_err)); 212 clear_err.address = phys; 213 clear_err.length = len; 214 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err, 215 sizeof(clear_err), &cmd_rc); 216 if (rc < 0) 217 return rc; 218 if (cmd_rc < 0) 219 return cmd_rc; 220 221 nvdimm_clear_from_poison_list(nvdimm_bus, phys, len); 222 return clear_err.cleared; 223 } 224 EXPORT_SYMBOL_GPL(nvdimm_clear_poison); 225 226 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv); 227 228 static struct bus_type nvdimm_bus_type = { 229 .name = "nd", 230 .uevent = nvdimm_bus_uevent, 231 .match = nvdimm_bus_match, 232 .probe = nvdimm_bus_probe, 233 .remove = nvdimm_bus_remove, 234 .shutdown = nvdimm_bus_shutdown, 235 }; 236 237 static void nvdimm_bus_release(struct device *dev) 238 { 239 struct nvdimm_bus *nvdimm_bus; 240 241 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); 242 ida_simple_remove(&nd_ida, nvdimm_bus->id); 243 kfree(nvdimm_bus); 244 } 245 246 static bool is_nvdimm_bus(struct device *dev) 247 { 248 return dev->release == nvdimm_bus_release; 249 } 250 251 struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev) 252 { 253 struct device *dev; 254 255 for (dev = nd_dev; dev; dev = dev->parent) 256 if (is_nvdimm_bus(dev)) 257 break; 258 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n"); 259 if (dev) 260 return to_nvdimm_bus(dev); 261 return NULL; 262 } 263 264 struct nvdimm_bus *to_nvdimm_bus(struct device *dev) 265 { 266 struct nvdimm_bus *nvdimm_bus; 267 268 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); 269 WARN_ON(!is_nvdimm_bus(dev)); 270 return nvdimm_bus; 271 } 272 EXPORT_SYMBOL_GPL(to_nvdimm_bus); 273 274 struct nvdimm_bus *nvdimm_bus_register(struct device *parent, 275 struct nvdimm_bus_descriptor *nd_desc) 276 { 277 struct nvdimm_bus *nvdimm_bus; 278 int rc; 279 280 nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL); 281 if (!nvdimm_bus) 282 return NULL; 283 INIT_LIST_HEAD(&nvdimm_bus->list); 284 INIT_LIST_HEAD(&nvdimm_bus->mapping_list); 285 INIT_LIST_HEAD(&nvdimm_bus->poison_list); 286 init_waitqueue_head(&nvdimm_bus->probe_wait); 287 nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL); 288 mutex_init(&nvdimm_bus->reconfig_mutex); 289 if (nvdimm_bus->id < 0) { 290 kfree(nvdimm_bus); 291 return NULL; 292 } 293 nvdimm_bus->nd_desc = nd_desc; 294 nvdimm_bus->dev.parent = parent; 295 nvdimm_bus->dev.release = nvdimm_bus_release; 296 nvdimm_bus->dev.groups = nd_desc->attr_groups; 297 nvdimm_bus->dev.bus = &nvdimm_bus_type; 298 dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id); 299 rc = device_register(&nvdimm_bus->dev); 300 if (rc) { 301 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc); 302 goto err; 303 } 304 305 return nvdimm_bus; 306 err: 307 put_device(&nvdimm_bus->dev); 308 return NULL; 309 } 310 EXPORT_SYMBOL_GPL(nvdimm_bus_register); 311 312 void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus) 313 { 314 if (!nvdimm_bus) 315 return; 316 device_unregister(&nvdimm_bus->dev); 317 } 318 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister); 319 320 static int child_unregister(struct device *dev, void *data) 321 { 322 /* 323 * the singular ndctl class device per bus needs to be 324 * "device_destroy"ed, so skip it here 325 * 326 * i.e. remove classless children 327 */ 328 if (dev->class) 329 /* pass */; 330 else 331 nd_device_unregister(dev, ND_SYNC); 332 return 0; 333 } 334 335 static void free_poison_list(struct list_head *poison_list) 336 { 337 struct nd_poison *pl, *next; 338 339 list_for_each_entry_safe(pl, next, poison_list, list) { 340 list_del(&pl->list); 341 kfree(pl); 342 } 343 list_del_init(poison_list); 344 } 345 346 static int nd_bus_remove(struct device *dev) 347 { 348 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 349 350 mutex_lock(&nvdimm_bus_list_mutex); 351 list_del_init(&nvdimm_bus->list); 352 mutex_unlock(&nvdimm_bus_list_mutex); 353 354 nd_synchronize(); 355 device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister); 356 357 nvdimm_bus_lock(&nvdimm_bus->dev); 358 free_poison_list(&nvdimm_bus->poison_list); 359 nvdimm_bus_unlock(&nvdimm_bus->dev); 360 361 nvdimm_bus_destroy_ndctl(nvdimm_bus); 362 363 return 0; 364 } 365 366 static int nd_bus_probe(struct device *dev) 367 { 368 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 369 int rc; 370 371 rc = nvdimm_bus_create_ndctl(nvdimm_bus); 372 if (rc) 373 return rc; 374 375 mutex_lock(&nvdimm_bus_list_mutex); 376 list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list); 377 mutex_unlock(&nvdimm_bus_list_mutex); 378 379 /* enable bus provider attributes to look up their local context */ 380 dev_set_drvdata(dev, nvdimm_bus->nd_desc); 381 382 return 0; 383 } 384 385 static struct nd_device_driver nd_bus_driver = { 386 .probe = nd_bus_probe, 387 .remove = nd_bus_remove, 388 .drv = { 389 .name = "nd_bus", 390 .suppress_bind_attrs = true, 391 .bus = &nvdimm_bus_type, 392 .owner = THIS_MODULE, 393 .mod_name = KBUILD_MODNAME, 394 }, 395 }; 396 397 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv) 398 { 399 struct nd_device_driver *nd_drv = to_nd_device_driver(drv); 400 401 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver) 402 return true; 403 404 return !!test_bit(to_nd_device_type(dev), &nd_drv->type); 405 } 406 407 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain); 408 409 void nd_synchronize(void) 410 { 411 async_synchronize_full_domain(&nd_async_domain); 412 } 413 EXPORT_SYMBOL_GPL(nd_synchronize); 414 415 static void nd_async_device_register(void *d, async_cookie_t cookie) 416 { 417 struct device *dev = d; 418 419 if (device_add(dev) != 0) { 420 dev_err(dev, "%s: failed\n", __func__); 421 put_device(dev); 422 } 423 put_device(dev); 424 } 425 426 static void nd_async_device_unregister(void *d, async_cookie_t cookie) 427 { 428 struct device *dev = d; 429 430 /* flush bus operations before delete */ 431 nvdimm_bus_lock(dev); 432 nvdimm_bus_unlock(dev); 433 434 device_unregister(dev); 435 put_device(dev); 436 } 437 438 void __nd_device_register(struct device *dev) 439 { 440 if (!dev) 441 return; 442 dev->bus = &nvdimm_bus_type; 443 get_device(dev); 444 async_schedule_domain(nd_async_device_register, dev, 445 &nd_async_domain); 446 } 447 448 void nd_device_register(struct device *dev) 449 { 450 device_initialize(dev); 451 __nd_device_register(dev); 452 } 453 EXPORT_SYMBOL(nd_device_register); 454 455 void nd_device_unregister(struct device *dev, enum nd_async_mode mode) 456 { 457 switch (mode) { 458 case ND_ASYNC: 459 get_device(dev); 460 async_schedule_domain(nd_async_device_unregister, dev, 461 &nd_async_domain); 462 break; 463 case ND_SYNC: 464 nd_synchronize(); 465 device_unregister(dev); 466 break; 467 } 468 } 469 EXPORT_SYMBOL(nd_device_unregister); 470 471 /** 472 * __nd_driver_register() - register a region or a namespace driver 473 * @nd_drv: driver to register 474 * @owner: automatically set by nd_driver_register() macro 475 * @mod_name: automatically set by nd_driver_register() macro 476 */ 477 int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner, 478 const char *mod_name) 479 { 480 struct device_driver *drv = &nd_drv->drv; 481 482 if (!nd_drv->type) { 483 pr_debug("driver type bitmask not set (%pf)\n", 484 __builtin_return_address(0)); 485 return -EINVAL; 486 } 487 488 if (!nd_drv->probe) { 489 pr_debug("%s ->probe() must be specified\n", mod_name); 490 return -EINVAL; 491 } 492 493 drv->bus = &nvdimm_bus_type; 494 drv->owner = owner; 495 drv->mod_name = mod_name; 496 497 return driver_register(drv); 498 } 499 EXPORT_SYMBOL(__nd_driver_register); 500 501 int nvdimm_revalidate_disk(struct gendisk *disk) 502 { 503 struct device *dev = disk_to_dev(disk)->parent; 504 struct nd_region *nd_region = to_nd_region(dev->parent); 505 const char *pol = nd_region->ro ? "only" : "write"; 506 507 if (nd_region->ro == get_disk_ro(disk)) 508 return 0; 509 510 dev_info(dev, "%s read-%s, marking %s read-%s\n", 511 dev_name(&nd_region->dev), pol, disk->disk_name, pol); 512 set_disk_ro(disk, nd_region->ro); 513 514 return 0; 515 516 } 517 EXPORT_SYMBOL(nvdimm_revalidate_disk); 518 519 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, 520 char *buf) 521 { 522 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n", 523 to_nd_device_type(dev)); 524 } 525 static DEVICE_ATTR_RO(modalias); 526 527 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr, 528 char *buf) 529 { 530 return sprintf(buf, "%s\n", dev->type->name); 531 } 532 static DEVICE_ATTR_RO(devtype); 533 534 static struct attribute *nd_device_attributes[] = { 535 &dev_attr_modalias.attr, 536 &dev_attr_devtype.attr, 537 NULL, 538 }; 539 540 /** 541 * nd_device_attribute_group - generic attributes for all devices on an nd bus 542 */ 543 struct attribute_group nd_device_attribute_group = { 544 .attrs = nd_device_attributes, 545 }; 546 EXPORT_SYMBOL_GPL(nd_device_attribute_group); 547 548 static ssize_t numa_node_show(struct device *dev, 549 struct device_attribute *attr, char *buf) 550 { 551 return sprintf(buf, "%d\n", dev_to_node(dev)); 552 } 553 static DEVICE_ATTR_RO(numa_node); 554 555 static struct attribute *nd_numa_attributes[] = { 556 &dev_attr_numa_node.attr, 557 NULL, 558 }; 559 560 static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a, 561 int n) 562 { 563 if (!IS_ENABLED(CONFIG_NUMA)) 564 return 0; 565 566 return a->mode; 567 } 568 569 /** 570 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus 571 */ 572 struct attribute_group nd_numa_attribute_group = { 573 .attrs = nd_numa_attributes, 574 .is_visible = nd_numa_attr_visible, 575 }; 576 EXPORT_SYMBOL_GPL(nd_numa_attribute_group); 577 578 int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus) 579 { 580 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id); 581 struct device *dev; 582 583 dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus, 584 "ndctl%d", nvdimm_bus->id); 585 586 if (IS_ERR(dev)) 587 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n", 588 nvdimm_bus->id, PTR_ERR(dev)); 589 return PTR_ERR_OR_ZERO(dev); 590 } 591 592 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus) 593 { 594 device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id)); 595 } 596 597 static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = { 598 [ND_CMD_IMPLEMENTED] = { }, 599 [ND_CMD_SMART] = { 600 .out_num = 2, 601 .out_sizes = { 4, 128, }, 602 }, 603 [ND_CMD_SMART_THRESHOLD] = { 604 .out_num = 2, 605 .out_sizes = { 4, 8, }, 606 }, 607 [ND_CMD_DIMM_FLAGS] = { 608 .out_num = 2, 609 .out_sizes = { 4, 4 }, 610 }, 611 [ND_CMD_GET_CONFIG_SIZE] = { 612 .out_num = 3, 613 .out_sizes = { 4, 4, 4, }, 614 }, 615 [ND_CMD_GET_CONFIG_DATA] = { 616 .in_num = 2, 617 .in_sizes = { 4, 4, }, 618 .out_num = 2, 619 .out_sizes = { 4, UINT_MAX, }, 620 }, 621 [ND_CMD_SET_CONFIG_DATA] = { 622 .in_num = 3, 623 .in_sizes = { 4, 4, UINT_MAX, }, 624 .out_num = 1, 625 .out_sizes = { 4, }, 626 }, 627 [ND_CMD_VENDOR] = { 628 .in_num = 3, 629 .in_sizes = { 4, 4, UINT_MAX, }, 630 .out_num = 3, 631 .out_sizes = { 4, 4, UINT_MAX, }, 632 }, 633 [ND_CMD_CALL] = { 634 .in_num = 2, 635 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, }, 636 .out_num = 1, 637 .out_sizes = { UINT_MAX, }, 638 }, 639 }; 640 641 const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd) 642 { 643 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs)) 644 return &__nd_cmd_dimm_descs[cmd]; 645 return NULL; 646 } 647 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc); 648 649 static const struct nd_cmd_desc __nd_cmd_bus_descs[] = { 650 [ND_CMD_IMPLEMENTED] = { }, 651 [ND_CMD_ARS_CAP] = { 652 .in_num = 2, 653 .in_sizes = { 8, 8, }, 654 .out_num = 4, 655 .out_sizes = { 4, 4, 4, 4, }, 656 }, 657 [ND_CMD_ARS_START] = { 658 .in_num = 5, 659 .in_sizes = { 8, 8, 2, 1, 5, }, 660 .out_num = 2, 661 .out_sizes = { 4, 4, }, 662 }, 663 [ND_CMD_ARS_STATUS] = { 664 .out_num = 3, 665 .out_sizes = { 4, 4, UINT_MAX, }, 666 }, 667 [ND_CMD_CLEAR_ERROR] = { 668 .in_num = 2, 669 .in_sizes = { 8, 8, }, 670 .out_num = 3, 671 .out_sizes = { 4, 4, 8, }, 672 }, 673 [ND_CMD_CALL] = { 674 .in_num = 2, 675 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, }, 676 .out_num = 1, 677 .out_sizes = { UINT_MAX, }, 678 }, 679 }; 680 681 const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd) 682 { 683 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs)) 684 return &__nd_cmd_bus_descs[cmd]; 685 return NULL; 686 } 687 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc); 688 689 u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd, 690 const struct nd_cmd_desc *desc, int idx, void *buf) 691 { 692 if (idx >= desc->in_num) 693 return UINT_MAX; 694 695 if (desc->in_sizes[idx] < UINT_MAX) 696 return desc->in_sizes[idx]; 697 698 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) { 699 struct nd_cmd_set_config_hdr *hdr = buf; 700 701 return hdr->in_length; 702 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) { 703 struct nd_cmd_vendor_hdr *hdr = buf; 704 705 return hdr->in_length; 706 } else if (cmd == ND_CMD_CALL) { 707 struct nd_cmd_pkg *pkg = buf; 708 709 return pkg->nd_size_in; 710 } 711 712 return UINT_MAX; 713 } 714 EXPORT_SYMBOL_GPL(nd_cmd_in_size); 715 716 u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd, 717 const struct nd_cmd_desc *desc, int idx, const u32 *in_field, 718 const u32 *out_field, unsigned long remainder) 719 { 720 if (idx >= desc->out_num) 721 return UINT_MAX; 722 723 if (desc->out_sizes[idx] < UINT_MAX) 724 return desc->out_sizes[idx]; 725 726 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1) 727 return in_field[1]; 728 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) 729 return out_field[1]; 730 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) { 731 /* 732 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is 733 * "Size of Output Buffer in bytes, including this 734 * field." 735 */ 736 if (out_field[1] < 4) 737 return 0; 738 /* 739 * ACPI 6.1 is ambiguous if 'status' is included in the 740 * output size. If we encounter an output size that 741 * overshoots the remainder by 4 bytes, assume it was 742 * including 'status'. 743 */ 744 if (out_field[1] - 8 == remainder) 745 return remainder; 746 return out_field[1] - 4; 747 } else if (cmd == ND_CMD_CALL) { 748 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field; 749 750 return pkg->nd_size_out; 751 } 752 753 754 return UINT_MAX; 755 } 756 EXPORT_SYMBOL_GPL(nd_cmd_out_size); 757 758 void wait_nvdimm_bus_probe_idle(struct device *dev) 759 { 760 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 761 762 do { 763 if (nvdimm_bus->probe_active == 0) 764 break; 765 nvdimm_bus_unlock(&nvdimm_bus->dev); 766 wait_event(nvdimm_bus->probe_wait, 767 nvdimm_bus->probe_active == 0); 768 nvdimm_bus_lock(&nvdimm_bus->dev); 769 } while (true); 770 } 771 772 static int pmem_active(struct device *dev, void *data) 773 { 774 if (is_nd_pmem(dev) && dev->driver) 775 return -EBUSY; 776 return 0; 777 } 778 779 /* set_config requires an idle interleave set */ 780 static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus, 781 struct nvdimm *nvdimm, unsigned int cmd) 782 { 783 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc; 784 785 /* ask the bus provider if it would like to block this request */ 786 if (nd_desc->clear_to_send) { 787 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd); 788 789 if (rc) 790 return rc; 791 } 792 793 /* require clear error to go through the pmem driver */ 794 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR) 795 return device_for_each_child(&nvdimm_bus->dev, NULL, 796 pmem_active); 797 798 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA) 799 return 0; 800 801 /* prevent label manipulation while the kernel owns label updates */ 802 wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev); 803 if (atomic_read(&nvdimm->busy)) 804 return -EBUSY; 805 return 0; 806 } 807 808 static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm, 809 int read_only, unsigned int ioctl_cmd, unsigned long arg) 810 { 811 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc; 812 size_t buf_len = 0, in_len = 0, out_len = 0; 813 static char out_env[ND_CMD_MAX_ENVELOPE]; 814 static char in_env[ND_CMD_MAX_ENVELOPE]; 815 const struct nd_cmd_desc *desc = NULL; 816 unsigned int cmd = _IOC_NR(ioctl_cmd); 817 void __user *p = (void __user *) arg; 818 struct device *dev = &nvdimm_bus->dev; 819 struct nd_cmd_pkg pkg; 820 const char *cmd_name, *dimm_name; 821 unsigned long cmd_mask; 822 void *buf; 823 int rc, i; 824 825 if (nvdimm) { 826 desc = nd_cmd_dimm_desc(cmd); 827 cmd_name = nvdimm_cmd_name(cmd); 828 cmd_mask = nvdimm->cmd_mask; 829 dimm_name = dev_name(&nvdimm->dev); 830 } else { 831 desc = nd_cmd_bus_desc(cmd); 832 cmd_name = nvdimm_bus_cmd_name(cmd); 833 cmd_mask = nd_desc->cmd_mask; 834 dimm_name = "bus"; 835 } 836 837 if (cmd == ND_CMD_CALL) { 838 if (copy_from_user(&pkg, p, sizeof(pkg))) 839 return -EFAULT; 840 } 841 842 if (!desc || (desc->out_num + desc->in_num == 0) || 843 !test_bit(cmd, &cmd_mask)) 844 return -ENOTTY; 845 846 /* fail write commands (when read-only) */ 847 if (read_only) 848 switch (cmd) { 849 case ND_CMD_VENDOR: 850 case ND_CMD_SET_CONFIG_DATA: 851 case ND_CMD_ARS_START: 852 case ND_CMD_CLEAR_ERROR: 853 case ND_CMD_CALL: 854 dev_dbg(&nvdimm_bus->dev, "'%s' command while read-only.\n", 855 nvdimm ? nvdimm_cmd_name(cmd) 856 : nvdimm_bus_cmd_name(cmd)); 857 return -EPERM; 858 default: 859 break; 860 } 861 862 /* process an input envelope */ 863 for (i = 0; i < desc->in_num; i++) { 864 u32 in_size, copy; 865 866 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env); 867 if (in_size == UINT_MAX) { 868 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n", 869 __func__, dimm_name, cmd_name, i); 870 return -ENXIO; 871 } 872 if (in_len < sizeof(in_env)) 873 copy = min_t(u32, sizeof(in_env) - in_len, in_size); 874 else 875 copy = 0; 876 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) 877 return -EFAULT; 878 in_len += in_size; 879 } 880 881 if (cmd == ND_CMD_CALL) { 882 dev_dbg(dev, "%s:%s, idx: %llu, in: %zu, out: %zu, len %zu\n", 883 __func__, dimm_name, pkg.nd_command, 884 in_len, out_len, buf_len); 885 886 for (i = 0; i < ARRAY_SIZE(pkg.nd_reserved2); i++) 887 if (pkg.nd_reserved2[i]) 888 return -EINVAL; 889 } 890 891 /* process an output envelope */ 892 for (i = 0; i < desc->out_num; i++) { 893 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, 894 (u32 *) in_env, (u32 *) out_env, 0); 895 u32 copy; 896 897 if (out_size == UINT_MAX) { 898 dev_dbg(dev, "%s:%s unknown output size cmd: %s field: %d\n", 899 __func__, dimm_name, cmd_name, i); 900 return -EFAULT; 901 } 902 if (out_len < sizeof(out_env)) 903 copy = min_t(u32, sizeof(out_env) - out_len, out_size); 904 else 905 copy = 0; 906 if (copy && copy_from_user(&out_env[out_len], 907 p + in_len + out_len, copy)) 908 return -EFAULT; 909 out_len += out_size; 910 } 911 912 buf_len = out_len + in_len; 913 if (buf_len > ND_IOCTL_MAX_BUFLEN) { 914 dev_dbg(dev, "%s:%s cmd: %s buf_len: %zu > %d\n", __func__, 915 dimm_name, cmd_name, buf_len, 916 ND_IOCTL_MAX_BUFLEN); 917 return -EINVAL; 918 } 919 920 buf = vmalloc(buf_len); 921 if (!buf) 922 return -ENOMEM; 923 924 if (copy_from_user(buf, p, buf_len)) { 925 rc = -EFAULT; 926 goto out; 927 } 928 929 nvdimm_bus_lock(&nvdimm_bus->dev); 930 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, cmd); 931 if (rc) 932 goto out_unlock; 933 934 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, NULL); 935 if (rc < 0) 936 goto out_unlock; 937 if (copy_to_user(p, buf, buf_len)) 938 rc = -EFAULT; 939 out_unlock: 940 nvdimm_bus_unlock(&nvdimm_bus->dev); 941 out: 942 vfree(buf); 943 return rc; 944 } 945 946 static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 947 { 948 long id = (long) file->private_data; 949 int rc = -ENXIO, ro; 950 struct nvdimm_bus *nvdimm_bus; 951 952 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY); 953 mutex_lock(&nvdimm_bus_list_mutex); 954 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) { 955 if (nvdimm_bus->id == id) { 956 rc = __nd_ioctl(nvdimm_bus, NULL, ro, cmd, arg); 957 break; 958 } 959 } 960 mutex_unlock(&nvdimm_bus_list_mutex); 961 962 return rc; 963 } 964 965 static int match_dimm(struct device *dev, void *data) 966 { 967 long id = (long) data; 968 969 if (is_nvdimm(dev)) { 970 struct nvdimm *nvdimm = to_nvdimm(dev); 971 972 return nvdimm->id == id; 973 } 974 975 return 0; 976 } 977 978 static long nvdimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 979 { 980 int rc = -ENXIO, ro; 981 struct nvdimm_bus *nvdimm_bus; 982 983 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY); 984 mutex_lock(&nvdimm_bus_list_mutex); 985 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) { 986 struct device *dev = device_find_child(&nvdimm_bus->dev, 987 file->private_data, match_dimm); 988 struct nvdimm *nvdimm; 989 990 if (!dev) 991 continue; 992 993 nvdimm = to_nvdimm(dev); 994 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg); 995 put_device(dev); 996 break; 997 } 998 mutex_unlock(&nvdimm_bus_list_mutex); 999 1000 return rc; 1001 } 1002 1003 static int nd_open(struct inode *inode, struct file *file) 1004 { 1005 long minor = iminor(inode); 1006 1007 file->private_data = (void *) minor; 1008 return 0; 1009 } 1010 1011 static const struct file_operations nvdimm_bus_fops = { 1012 .owner = THIS_MODULE, 1013 .open = nd_open, 1014 .unlocked_ioctl = nd_ioctl, 1015 .compat_ioctl = nd_ioctl, 1016 .llseek = noop_llseek, 1017 }; 1018 1019 static const struct file_operations nvdimm_fops = { 1020 .owner = THIS_MODULE, 1021 .open = nd_open, 1022 .unlocked_ioctl = nvdimm_ioctl, 1023 .compat_ioctl = nvdimm_ioctl, 1024 .llseek = noop_llseek, 1025 }; 1026 1027 int __init nvdimm_bus_init(void) 1028 { 1029 int rc; 1030 1031 BUILD_BUG_ON(sizeof(struct nd_smart_payload) != 128); 1032 BUILD_BUG_ON(sizeof(struct nd_smart_threshold_payload) != 8); 1033 1034 rc = bus_register(&nvdimm_bus_type); 1035 if (rc) 1036 return rc; 1037 1038 rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops); 1039 if (rc < 0) 1040 goto err_bus_chrdev; 1041 nvdimm_bus_major = rc; 1042 1043 rc = register_chrdev(0, "dimmctl", &nvdimm_fops); 1044 if (rc < 0) 1045 goto err_dimm_chrdev; 1046 nvdimm_major = rc; 1047 1048 nd_class = class_create(THIS_MODULE, "nd"); 1049 if (IS_ERR(nd_class)) { 1050 rc = PTR_ERR(nd_class); 1051 goto err_class; 1052 } 1053 1054 rc = driver_register(&nd_bus_driver.drv); 1055 if (rc) 1056 goto err_nd_bus; 1057 1058 return 0; 1059 1060 err_nd_bus: 1061 class_destroy(nd_class); 1062 err_class: 1063 unregister_chrdev(nvdimm_major, "dimmctl"); 1064 err_dimm_chrdev: 1065 unregister_chrdev(nvdimm_bus_major, "ndctl"); 1066 err_bus_chrdev: 1067 bus_unregister(&nvdimm_bus_type); 1068 1069 return rc; 1070 } 1071 1072 void nvdimm_bus_exit(void) 1073 { 1074 driver_unregister(&nd_bus_driver.drv); 1075 class_destroy(nd_class); 1076 unregister_chrdev(nvdimm_bus_major, "ndctl"); 1077 unregister_chrdev(nvdimm_major, "dimmctl"); 1078 bus_unregister(&nvdimm_bus_type); 1079 ida_destroy(&nd_ida); 1080 } 1081