1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ 3 #include <linux/libnvdimm.h> 4 #include <linux/unaligned.h> 5 #include <linux/device.h> 6 #include <linux/module.h> 7 #include <linux/ndctl.h> 8 #include <linux/async.h> 9 #include <linux/slab.h> 10 #include <linux/nd.h> 11 #include "cxlmem.h" 12 #include "cxl.h" 13 14 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); 15 16 /** 17 * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology 18 * @host: platform firmware root device 19 * @port: CXL port at the root of a CXL topology 20 * 21 * Return: bridge device that can host cxl_nvdimm objects 22 */ 23 struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, 24 struct cxl_port *port) 25 { 26 return __devm_cxl_add_nvdimm_bridge(host, port); 27 } 28 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL"); 29 30 static void clear_exclusive(void *mds) 31 { 32 clear_exclusive_cxl_commands(mds, exclusive_cmds); 33 } 34 35 static void unregister_nvdimm(void *nvdimm) 36 { 37 nvdimm_delete(nvdimm); 38 } 39 40 static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf) 41 { 42 struct nvdimm *nvdimm = to_nvdimm(dev); 43 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 44 45 return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev)); 46 } 47 static DEVICE_ATTR_RO(provider); 48 49 static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf) 50 { 51 struct nvdimm *nvdimm = to_nvdimm(dev); 52 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 53 struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds; 54 55 return sysfs_emit(buf, "%lld\n", cxlds->serial); 56 } 57 static DEVICE_ATTR_RO(id); 58 59 static ssize_t dirty_shutdown_show(struct device *dev, 60 struct device_attribute *attr, char *buf) 61 { 62 struct nvdimm *nvdimm = to_nvdimm(dev); 63 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 64 65 return sysfs_emit(buf, "%llu\n", cxl_nvd->dirty_shutdowns); 66 } 67 static DEVICE_ATTR_RO(dirty_shutdown); 68 69 static struct attribute *cxl_dimm_attributes[] = { 70 &dev_attr_id.attr, 71 &dev_attr_provider.attr, 72 &dev_attr_dirty_shutdown.attr, 73 NULL 74 }; 75 76 #define CXL_INVALID_DIRTY_SHUTDOWN_COUNT ULLONG_MAX 77 static umode_t cxl_dimm_visible(struct kobject *kobj, 78 struct attribute *a, int n) 79 { 80 if (a == &dev_attr_dirty_shutdown.attr) { 81 struct device *dev = kobj_to_dev(kobj); 82 struct nvdimm *nvdimm = to_nvdimm(dev); 83 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 84 85 if (cxl_nvd->dirty_shutdowns == 86 CXL_INVALID_DIRTY_SHUTDOWN_COUNT) 87 return 0; 88 } 89 90 return a->mode; 91 } 92 93 static const struct attribute_group cxl_dimm_attribute_group = { 94 .name = "cxl", 95 .attrs = cxl_dimm_attributes, 96 .is_visible = cxl_dimm_visible 97 }; 98 99 static const struct attribute_group *cxl_dimm_attribute_groups[] = { 100 &cxl_dimm_attribute_group, 101 NULL 102 }; 103 104 static void cxl_nvdimm_arm_dirty_shutdown_tracking(struct cxl_nvdimm *cxl_nvd) 105 { 106 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; 107 struct cxl_dev_state *cxlds = cxlmd->cxlds; 108 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 109 struct device *dev = &cxl_nvd->dev; 110 u32 count; 111 112 /* 113 * Dirty tracking is enabled and exposed to the user, only when: 114 * - dirty shutdown on the device can be set, and, 115 * - the device has a Device GPF DVSEC (albeit unused), and, 116 * - the Get Health Info cmd can retrieve the device's dirty count. 117 */ 118 cxl_nvd->dirty_shutdowns = CXL_INVALID_DIRTY_SHUTDOWN_COUNT; 119 120 if (cxl_arm_dirty_shutdown(mds)) { 121 dev_warn(dev, "GPF: could not set dirty shutdown state\n"); 122 return; 123 } 124 125 if (!cxl_gpf_get_dvsec(cxlds->dev)) 126 return; 127 128 if (cxl_get_dirty_count(mds, &count)) { 129 dev_warn(dev, "GPF: could not retrieve dirty count\n"); 130 return; 131 } 132 133 cxl_nvd->dirty_shutdowns = count; 134 } 135 136 static int cxl_nvdimm_probe(struct device *dev) 137 { 138 struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev); 139 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; 140 struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb; 141 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 142 unsigned long flags = 0, cmd_mask = 0; 143 struct nvdimm *nvdimm; 144 int rc; 145 146 if (test_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags)) 147 return -EBUSY; 148 149 set_exclusive_cxl_commands(mds, exclusive_cmds); 150 rc = devm_add_action_or_reset(dev, clear_exclusive, mds); 151 if (rc) 152 return rc; 153 154 set_bit(NDD_LABELING, &flags); 155 set_bit(NDD_REGISTER_SYNC, &flags); 156 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask); 157 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask); 158 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask); 159 160 /* 161 * Set dirty shutdown now, with the expectation that the device 162 * clear it upon a successful GPF flow. The exception to this 163 * is upon Viral detection, per CXL 3.2 section 12.4.2. 164 */ 165 cxl_nvdimm_arm_dirty_shutdown_tracking(cxl_nvd); 166 167 nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, 168 cxl_dimm_attribute_groups, flags, 169 cmd_mask, 0, NULL, cxl_nvd->dev_id, 170 cxl_security_ops, NULL); 171 if (!nvdimm) 172 return -ENOMEM; 173 174 dev_set_drvdata(dev, nvdimm); 175 return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm); 176 } 177 178 static struct cxl_driver cxl_nvdimm_driver = { 179 .name = "cxl_nvdimm", 180 .probe = cxl_nvdimm_probe, 181 .id = CXL_DEVICE_NVDIMM, 182 .drv = { 183 .suppress_bind_attrs = true, 184 }, 185 }; 186 187 static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds, 188 struct nd_cmd_get_config_size *cmd, 189 unsigned int buf_len) 190 { 191 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 192 193 if (sizeof(*cmd) > buf_len) 194 return -EINVAL; 195 196 *cmd = (struct nd_cmd_get_config_size){ 197 .config_size = mds->lsa_size, 198 .max_xfer = 199 cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa), 200 }; 201 202 return 0; 203 } 204 205 static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds, 206 struct nd_cmd_get_config_data_hdr *cmd, 207 unsigned int buf_len) 208 { 209 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 210 struct cxl_mbox_get_lsa get_lsa; 211 struct cxl_mbox_cmd mbox_cmd; 212 int rc; 213 214 if (sizeof(*cmd) > buf_len) 215 return -EINVAL; 216 if (struct_size(cmd, out_buf, cmd->in_length) > buf_len) 217 return -EINVAL; 218 219 get_lsa = (struct cxl_mbox_get_lsa) { 220 .offset = cpu_to_le32(cmd->in_offset), 221 .length = cpu_to_le32(cmd->in_length), 222 }; 223 mbox_cmd = (struct cxl_mbox_cmd) { 224 .opcode = CXL_MBOX_OP_GET_LSA, 225 .payload_in = &get_lsa, 226 .size_in = sizeof(get_lsa), 227 .size_out = cmd->in_length, 228 .payload_out = cmd->out_buf, 229 }; 230 231 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 232 cmd->status = 0; 233 234 return rc; 235 } 236 237 static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds, 238 struct nd_cmd_set_config_hdr *cmd, 239 unsigned int buf_len) 240 { 241 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 242 struct cxl_mbox_set_lsa *set_lsa; 243 struct cxl_mbox_cmd mbox_cmd; 244 int rc; 245 246 if (sizeof(*cmd) > buf_len) 247 return -EINVAL; 248 249 /* 4-byte status follows the input data in the payload */ 250 if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len) 251 return -EINVAL; 252 253 set_lsa = 254 kvzalloc_flex(*set_lsa, data, cmd->in_length); 255 if (!set_lsa) 256 return -ENOMEM; 257 258 *set_lsa = (struct cxl_mbox_set_lsa) { 259 .offset = cpu_to_le32(cmd->in_offset), 260 }; 261 memcpy(set_lsa->data, cmd->in_buf, cmd->in_length); 262 mbox_cmd = (struct cxl_mbox_cmd) { 263 .opcode = CXL_MBOX_OP_SET_LSA, 264 .payload_in = set_lsa, 265 .size_in = struct_size(set_lsa, data, cmd->in_length), 266 }; 267 268 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 269 270 /* 271 * Set "firmware" status (4-packed bytes at the end of the input 272 * payload. 273 */ 274 put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]); 275 kvfree(set_lsa); 276 277 return rc; 278 } 279 280 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd, 281 void *buf, unsigned int buf_len) 282 { 283 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 284 unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm); 285 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; 286 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 287 288 if (!test_bit(cmd, &cmd_mask)) 289 return -ENOTTY; 290 291 switch (cmd) { 292 case ND_CMD_GET_CONFIG_SIZE: 293 return cxl_pmem_get_config_size(mds, buf, buf_len); 294 case ND_CMD_GET_CONFIG_DATA: 295 return cxl_pmem_get_config_data(mds, buf, buf_len); 296 case ND_CMD_SET_CONFIG_DATA: 297 return cxl_pmem_set_config_data(mds, buf, buf_len); 298 default: 299 return -ENOTTY; 300 } 301 } 302 303 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc, 304 struct nvdimm *nvdimm, unsigned int cmd, void *buf, 305 unsigned int buf_len, int *cmd_rc) 306 { 307 /* 308 * No firmware response to translate, let the transport error 309 * code take precedence. 310 */ 311 *cmd_rc = 0; 312 313 if (!nvdimm) 314 return -ENOTTY; 315 return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len); 316 } 317 318 static int detach_nvdimm(struct device *dev, void *data) 319 { 320 struct cxl_nvdimm *cxl_nvd; 321 bool release = false; 322 323 if (!is_cxl_nvdimm(dev)) 324 return 0; 325 326 scoped_guard(device, dev) { 327 if (dev->driver) { 328 cxl_nvd = to_cxl_nvdimm(dev); 329 if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) { 330 release = true; 331 set_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags); 332 } 333 } 334 } 335 if (release) 336 device_release_driver(dev); 337 return 0; 338 } 339 340 static void unregister_nvdimm_bus(void *_cxl_nvb) 341 { 342 struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb; 343 struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus; 344 345 bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm); 346 347 cxl_nvb->nvdimm_bus = NULL; 348 nvdimm_bus_unregister(nvdimm_bus); 349 } 350 351 static int cxl_nvdimm_bridge_probe(struct device *dev) 352 { 353 struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev); 354 355 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) { 356 .provider_name = "CXL", 357 .module = THIS_MODULE, 358 .ndctl = cxl_pmem_ctl, 359 }; 360 361 cxl_nvb->nvdimm_bus = 362 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc); 363 364 if (!cxl_nvb->nvdimm_bus) 365 return -ENOMEM; 366 367 return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb); 368 } 369 370 static struct cxl_driver cxl_nvdimm_bridge_driver = { 371 .name = "cxl_nvdimm_bridge", 372 .probe = cxl_nvdimm_bridge_probe, 373 .id = CXL_DEVICE_NVDIMM_BRIDGE, 374 .drv = { 375 .probe_type = PROBE_FORCE_SYNCHRONOUS, 376 .suppress_bind_attrs = true, 377 }, 378 }; 379 380 static void unregister_nvdimm_region(void *nd_region) 381 { 382 nvdimm_region_delete(nd_region); 383 } 384 385 static void cxlr_pmem_remove_resource(void *res) 386 { 387 remove_resource(res); 388 } 389 390 struct cxl_pmem_region_info { 391 u64 offset; 392 u64 serial; 393 }; 394 395 static int cxl_pmem_region_probe(struct device *dev) 396 { 397 struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE]; 398 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 399 struct cxl_region *cxlr = cxlr_pmem->cxlr; 400 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 401 struct cxl_pmem_region_info *info = NULL; 402 struct nd_interleave_set *nd_set; 403 struct nd_region_desc ndr_desc; 404 struct cxl_nvdimm *cxl_nvd; 405 struct nvdimm *nvdimm; 406 struct resource *res; 407 int rc, i = 0; 408 409 memset(&mappings, 0, sizeof(mappings)); 410 memset(&ndr_desc, 0, sizeof(ndr_desc)); 411 412 res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL); 413 if (!res) 414 return -ENOMEM; 415 416 res->name = "Persistent Memory"; 417 res->start = cxlr_pmem->hpa_range.start; 418 res->end = cxlr_pmem->hpa_range.end; 419 res->flags = IORESOURCE_MEM; 420 res->desc = IORES_DESC_PERSISTENT_MEMORY; 421 422 rc = insert_resource(&iomem_resource, res); 423 if (rc) 424 return rc; 425 426 rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res); 427 if (rc) 428 return rc; 429 430 ndr_desc.res = res; 431 ndr_desc.provider_data = cxlr_pmem; 432 433 ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start); 434 ndr_desc.target_node = phys_to_target_node(res->start); 435 if (ndr_desc.target_node == NUMA_NO_NODE) { 436 ndr_desc.target_node = ndr_desc.numa_node; 437 dev_dbg(&cxlr->dev, "changing target node from %d to %d", 438 NUMA_NO_NODE, ndr_desc.target_node); 439 } 440 441 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 442 if (!nd_set) 443 return -ENOMEM; 444 445 ndr_desc.memregion = cxlr->id; 446 set_bit(ND_REGION_CXL, &ndr_desc.flags); 447 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags); 448 449 info = kmalloc_objs(*info, cxlr_pmem->nr_mappings); 450 if (!info) 451 return -ENOMEM; 452 453 for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 454 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 455 struct cxl_memdev *cxlmd = m->cxlmd; 456 struct cxl_dev_state *cxlds = cxlmd->cxlds; 457 458 cxl_nvd = cxlmd->cxl_nvd; 459 nvdimm = dev_get_drvdata(&cxl_nvd->dev); 460 if (!nvdimm) { 461 dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i, 462 dev_name(&cxlmd->dev)); 463 rc = -ENODEV; 464 goto out_nvd; 465 } 466 467 if (cxlds->serial == 0) { 468 /* include missing alongside invalid in this error message. */ 469 dev_err(dev, "%s: invalid or missing serial number\n", 470 dev_name(&cxlmd->dev)); 471 rc = -ENXIO; 472 goto out_nvd; 473 } 474 info[i].serial = cxlds->serial; 475 info[i].offset = m->start; 476 477 m->cxl_nvd = cxl_nvd; 478 mappings[i] = (struct nd_mapping_desc) { 479 .nvdimm = nvdimm, 480 .start = m->start, 481 .size = m->size, 482 .position = i, 483 }; 484 } 485 ndr_desc.num_mappings = cxlr_pmem->nr_mappings; 486 ndr_desc.mapping = mappings; 487 488 /* 489 * TODO enable CXL labels which skip the need for 'interleave-set cookie' 490 */ 491 nd_set->cookie1 = 492 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0); 493 nd_set->cookie2 = nd_set->cookie1; 494 ndr_desc.nd_set = nd_set; 495 496 cxlr_pmem->nd_region = 497 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc); 498 if (!cxlr_pmem->nd_region) { 499 rc = -ENOMEM; 500 goto out_nvd; 501 } 502 503 rc = devm_add_action_or_reset(dev, unregister_nvdimm_region, 504 cxlr_pmem->nd_region); 505 out_nvd: 506 kfree(info); 507 508 return rc; 509 } 510 511 static struct cxl_driver cxl_pmem_region_driver = { 512 .name = "cxl_pmem_region", 513 .probe = cxl_pmem_region_probe, 514 .id = CXL_DEVICE_PMEM_REGION, 515 .drv = { 516 .suppress_bind_attrs = true, 517 }, 518 }; 519 520 static __init int cxl_pmem_init(void) 521 { 522 int rc; 523 524 set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds); 525 set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds); 526 527 rc = cxl_driver_register(&cxl_nvdimm_bridge_driver); 528 if (rc) 529 return rc; 530 531 rc = cxl_driver_register(&cxl_nvdimm_driver); 532 if (rc) 533 goto err_nvdimm; 534 535 rc = cxl_driver_register(&cxl_pmem_region_driver); 536 if (rc) 537 goto err_region; 538 539 return 0; 540 541 err_region: 542 cxl_driver_unregister(&cxl_nvdimm_driver); 543 err_nvdimm: 544 cxl_driver_unregister(&cxl_nvdimm_bridge_driver); 545 return rc; 546 } 547 548 static __exit void cxl_pmem_exit(void) 549 { 550 cxl_driver_unregister(&cxl_pmem_region_driver); 551 cxl_driver_unregister(&cxl_nvdimm_driver); 552 cxl_driver_unregister(&cxl_nvdimm_bridge_driver); 553 } 554 555 MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support"); 556 MODULE_LICENSE("GPL v2"); 557 module_init(cxl_pmem_init); 558 module_exit(cxl_pmem_exit); 559 MODULE_IMPORT_NS("CXL"); 560 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); 561 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM); 562 MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION); 563