1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 kfree(cxlmd); 31 } 32 33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 34 kgid_t *gid) 35 { 36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 37 } 38 39 static ssize_t firmware_version_show(struct device *dev, 40 struct device_attribute *attr, char *buf) 41 { 42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 43 struct cxl_dev_state *cxlds = cxlmd->cxlds; 44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 45 46 if (!mds) 47 return sysfs_emit(buf, "\n"); 48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 49 } 50 static DEVICE_ATTR_RO(firmware_version); 51 52 static ssize_t payload_max_show(struct device *dev, 53 struct device_attribute *attr, char *buf) 54 { 55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 56 struct cxl_dev_state *cxlds = cxlmd->cxlds; 57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 58 59 if (!mds) 60 return sysfs_emit(buf, "\n"); 61 return sysfs_emit(buf, "%zu\n", mds->payload_size); 62 } 63 static DEVICE_ATTR_RO(payload_max); 64 65 static ssize_t label_storage_size_show(struct device *dev, 66 struct device_attribute *attr, char *buf) 67 { 68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 69 struct cxl_dev_state *cxlds = cxlmd->cxlds; 70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 71 72 if (!mds) 73 return sysfs_emit(buf, "\n"); 74 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 75 } 76 static DEVICE_ATTR_RO(label_storage_size); 77 78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 79 char *buf) 80 { 81 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 82 struct cxl_dev_state *cxlds = cxlmd->cxlds; 83 unsigned long long len = resource_size(&cxlds->ram_res); 84 85 return sysfs_emit(buf, "%#llx\n", len); 86 } 87 88 static struct device_attribute dev_attr_ram_size = 89 __ATTR(size, 0444, ram_size_show, NULL); 90 91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 92 char *buf) 93 { 94 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 95 struct cxl_dev_state *cxlds = cxlmd->cxlds; 96 unsigned long long len = resource_size(&cxlds->pmem_res); 97 98 return sysfs_emit(buf, "%#llx\n", len); 99 } 100 101 static struct device_attribute dev_attr_pmem_size = 102 __ATTR(size, 0444, pmem_size_show, NULL); 103 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 105 char *buf) 106 { 107 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 108 struct cxl_dev_state *cxlds = cxlmd->cxlds; 109 110 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 111 } 112 static DEVICE_ATTR_RO(serial); 113 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 115 char *buf) 116 { 117 return sysfs_emit(buf, "%d\n", dev_to_node(dev)); 118 } 119 static DEVICE_ATTR_RO(numa_node); 120 121 static ssize_t security_state_show(struct device *dev, 122 struct device_attribute *attr, 123 char *buf) 124 { 125 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 126 struct cxl_dev_state *cxlds = cxlmd->cxlds; 127 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 128 unsigned long state = mds->security.state; 129 int rc = 0; 130 131 /* sync with latest submission state */ 132 mutex_lock(&mds->mbox_mutex); 133 if (mds->security.sanitize_active) 134 rc = sysfs_emit(buf, "sanitize\n"); 135 mutex_unlock(&mds->mbox_mutex); 136 if (rc) 137 return rc; 138 139 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 140 return sysfs_emit(buf, "disabled\n"); 141 if (state & CXL_PMEM_SEC_STATE_FROZEN || 142 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 143 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 144 return sysfs_emit(buf, "frozen\n"); 145 if (state & CXL_PMEM_SEC_STATE_LOCKED) 146 return sysfs_emit(buf, "locked\n"); 147 else 148 return sysfs_emit(buf, "unlocked\n"); 149 } 150 static struct device_attribute dev_attr_security_state = 151 __ATTR(state, 0444, security_state_show, NULL); 152 153 static ssize_t security_sanitize_store(struct device *dev, 154 struct device_attribute *attr, 155 const char *buf, size_t len) 156 { 157 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 158 bool sanitize; 159 ssize_t rc; 160 161 if (kstrtobool(buf, &sanitize) || !sanitize) 162 return -EINVAL; 163 164 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 165 if (rc) 166 return rc; 167 168 return len; 169 } 170 static struct device_attribute dev_attr_security_sanitize = 171 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 172 173 static ssize_t security_erase_store(struct device *dev, 174 struct device_attribute *attr, 175 const char *buf, size_t len) 176 { 177 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 178 ssize_t rc; 179 bool erase; 180 181 if (kstrtobool(buf, &erase) || !erase) 182 return -EINVAL; 183 184 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 185 if (rc) 186 return rc; 187 188 return len; 189 } 190 static struct device_attribute dev_attr_security_erase = 191 __ATTR(erase, 0200, NULL, security_erase_store); 192 193 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 194 { 195 struct cxl_dev_state *cxlds = cxlmd->cxlds; 196 u64 offset, length; 197 int rc = 0; 198 199 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 200 if (resource_size(&cxlds->pmem_res)) { 201 offset = cxlds->pmem_res.start; 202 length = resource_size(&cxlds->pmem_res); 203 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 204 if (rc) 205 return rc; 206 } 207 if (resource_size(&cxlds->ram_res)) { 208 offset = cxlds->ram_res.start; 209 length = resource_size(&cxlds->ram_res); 210 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 211 /* 212 * Invalid Physical Address is not an error for 213 * volatile addresses. Device support is optional. 214 */ 215 if (rc == -EFAULT) 216 rc = 0; 217 } 218 return rc; 219 } 220 221 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 222 { 223 struct cxl_port *port; 224 int rc; 225 226 port = cxlmd->endpoint; 227 if (!port || !is_cxl_endpoint(port)) 228 return -EINVAL; 229 230 rc = down_read_interruptible(&cxl_region_rwsem); 231 if (rc) 232 return rc; 233 234 rc = down_read_interruptible(&cxl_dpa_rwsem); 235 if (rc) { 236 up_read(&cxl_region_rwsem); 237 return rc; 238 } 239 240 if (cxl_num_decoders_committed(port) == 0) { 241 /* No regions mapped to this memdev */ 242 rc = cxl_get_poison_by_memdev(cxlmd); 243 } else { 244 /* Regions mapped, collect poison by endpoint */ 245 rc = cxl_get_poison_by_endpoint(port); 246 } 247 up_read(&cxl_dpa_rwsem); 248 up_read(&cxl_region_rwsem); 249 250 return rc; 251 } 252 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL); 253 254 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 255 { 256 struct cxl_dev_state *cxlds = cxlmd->cxlds; 257 258 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 259 return 0; 260 261 if (!resource_size(&cxlds->dpa_res)) { 262 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 263 return -EINVAL; 264 } 265 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 266 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 267 dpa, &cxlds->dpa_res); 268 return -EINVAL; 269 } 270 if (!IS_ALIGNED(dpa, 64)) { 271 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 272 return -EINVAL; 273 } 274 275 return 0; 276 } 277 278 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 279 { 280 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 281 struct cxl_mbox_inject_poison inject; 282 struct cxl_poison_record record; 283 struct cxl_mbox_cmd mbox_cmd; 284 struct cxl_region *cxlr; 285 int rc; 286 287 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 288 return 0; 289 290 rc = down_read_interruptible(&cxl_region_rwsem); 291 if (rc) 292 return rc; 293 294 rc = down_read_interruptible(&cxl_dpa_rwsem); 295 if (rc) { 296 up_read(&cxl_region_rwsem); 297 return rc; 298 } 299 300 rc = cxl_validate_poison_dpa(cxlmd, dpa); 301 if (rc) 302 goto out; 303 304 inject.address = cpu_to_le64(dpa); 305 mbox_cmd = (struct cxl_mbox_cmd) { 306 .opcode = CXL_MBOX_OP_INJECT_POISON, 307 .size_in = sizeof(inject), 308 .payload_in = &inject, 309 }; 310 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 311 if (rc) 312 goto out; 313 314 cxlr = cxl_dpa_to_region(cxlmd, dpa); 315 if (cxlr) 316 dev_warn_once(mds->cxlds.dev, 317 "poison inject dpa:%#llx region: %s\n", dpa, 318 dev_name(&cxlr->dev)); 319 320 record = (struct cxl_poison_record) { 321 .address = cpu_to_le64(dpa), 322 .length = cpu_to_le32(1), 323 }; 324 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 325 out: 326 up_read(&cxl_dpa_rwsem); 327 up_read(&cxl_region_rwsem); 328 329 return rc; 330 } 331 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL); 332 333 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 334 { 335 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 336 struct cxl_mbox_clear_poison clear; 337 struct cxl_poison_record record; 338 struct cxl_mbox_cmd mbox_cmd; 339 struct cxl_region *cxlr; 340 int rc; 341 342 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 343 return 0; 344 345 rc = down_read_interruptible(&cxl_region_rwsem); 346 if (rc) 347 return rc; 348 349 rc = down_read_interruptible(&cxl_dpa_rwsem); 350 if (rc) { 351 up_read(&cxl_region_rwsem); 352 return rc; 353 } 354 355 rc = cxl_validate_poison_dpa(cxlmd, dpa); 356 if (rc) 357 goto out; 358 359 /* 360 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 361 * is defined to accept 64 bytes of write-data, along with the 362 * address to clear. This driver uses zeroes as write-data. 363 */ 364 clear = (struct cxl_mbox_clear_poison) { 365 .address = cpu_to_le64(dpa) 366 }; 367 368 mbox_cmd = (struct cxl_mbox_cmd) { 369 .opcode = CXL_MBOX_OP_CLEAR_POISON, 370 .size_in = sizeof(clear), 371 .payload_in = &clear, 372 }; 373 374 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 375 if (rc) 376 goto out; 377 378 cxlr = cxl_dpa_to_region(cxlmd, dpa); 379 if (cxlr) 380 dev_warn_once(mds->cxlds.dev, 381 "poison clear dpa:%#llx region: %s\n", dpa, 382 dev_name(&cxlr->dev)); 383 384 record = (struct cxl_poison_record) { 385 .address = cpu_to_le64(dpa), 386 .length = cpu_to_le32(1), 387 }; 388 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 389 out: 390 up_read(&cxl_dpa_rwsem); 391 up_read(&cxl_region_rwsem); 392 393 return rc; 394 } 395 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL); 396 397 static struct attribute *cxl_memdev_attributes[] = { 398 &dev_attr_serial.attr, 399 &dev_attr_firmware_version.attr, 400 &dev_attr_payload_max.attr, 401 &dev_attr_label_storage_size.attr, 402 &dev_attr_numa_node.attr, 403 NULL, 404 }; 405 406 static ssize_t pmem_qos_class_show(struct device *dev, 407 struct device_attribute *attr, char *buf) 408 { 409 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 410 struct cxl_dev_state *cxlds = cxlmd->cxlds; 411 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 412 413 return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class); 414 } 415 416 static struct device_attribute dev_attr_pmem_qos_class = 417 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); 418 419 static struct attribute *cxl_memdev_pmem_attributes[] = { 420 &dev_attr_pmem_size.attr, 421 &dev_attr_pmem_qos_class.attr, 422 NULL, 423 }; 424 425 static ssize_t ram_qos_class_show(struct device *dev, 426 struct device_attribute *attr, char *buf) 427 { 428 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 429 struct cxl_dev_state *cxlds = cxlmd->cxlds; 430 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 431 432 return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class); 433 } 434 435 static struct device_attribute dev_attr_ram_qos_class = 436 __ATTR(qos_class, 0444, ram_qos_class_show, NULL); 437 438 static struct attribute *cxl_memdev_ram_attributes[] = { 439 &dev_attr_ram_size.attr, 440 &dev_attr_ram_qos_class.attr, 441 NULL, 442 }; 443 444 static struct attribute *cxl_memdev_security_attributes[] = { 445 &dev_attr_security_state.attr, 446 &dev_attr_security_sanitize.attr, 447 &dev_attr_security_erase.attr, 448 NULL, 449 }; 450 451 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 452 int n) 453 { 454 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 455 return 0; 456 return a->mode; 457 } 458 459 static struct attribute_group cxl_memdev_attribute_group = { 460 .attrs = cxl_memdev_attributes, 461 .is_visible = cxl_memdev_visible, 462 }; 463 464 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) 465 { 466 struct device *dev = kobj_to_dev(kobj); 467 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 468 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 469 470 if (a == &dev_attr_ram_qos_class.attr) 471 if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID) 472 return 0; 473 474 return a->mode; 475 } 476 477 static struct attribute_group cxl_memdev_ram_attribute_group = { 478 .name = "ram", 479 .attrs = cxl_memdev_ram_attributes, 480 .is_visible = cxl_ram_visible, 481 }; 482 483 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) 484 { 485 struct device *dev = kobj_to_dev(kobj); 486 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 487 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 488 489 if (a == &dev_attr_pmem_qos_class.attr) 490 if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID) 491 return 0; 492 493 return a->mode; 494 } 495 496 static struct attribute_group cxl_memdev_pmem_attribute_group = { 497 .name = "pmem", 498 .attrs = cxl_memdev_pmem_attributes, 499 .is_visible = cxl_pmem_visible, 500 }; 501 502 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 503 struct attribute *a, int n) 504 { 505 struct device *dev = kobj_to_dev(kobj); 506 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 507 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 508 509 if (a == &dev_attr_security_sanitize.attr && 510 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 511 return 0; 512 513 if (a == &dev_attr_security_erase.attr && 514 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 515 return 0; 516 517 return a->mode; 518 } 519 520 static struct attribute_group cxl_memdev_security_attribute_group = { 521 .name = "security", 522 .attrs = cxl_memdev_security_attributes, 523 .is_visible = cxl_memdev_security_visible, 524 }; 525 526 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 527 &cxl_memdev_attribute_group, 528 &cxl_memdev_ram_attribute_group, 529 &cxl_memdev_pmem_attribute_group, 530 &cxl_memdev_security_attribute_group, 531 NULL, 532 }; 533 534 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) 535 { 536 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); 537 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); 538 } 539 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, CXL); 540 541 static const struct device_type cxl_memdev_type = { 542 .name = "cxl_memdev", 543 .release = cxl_memdev_release, 544 .devnode = cxl_memdev_devnode, 545 .groups = cxl_memdev_attribute_groups, 546 }; 547 548 bool is_cxl_memdev(const struct device *dev) 549 { 550 return dev->type == &cxl_memdev_type; 551 } 552 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL); 553 554 /** 555 * set_exclusive_cxl_commands() - atomically disable user cxl commands 556 * @mds: The device state to operate on 557 * @cmds: bitmap of commands to mark exclusive 558 * 559 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 560 * invocations of the ioctl path and then disable future execution of 561 * commands with the command ids set in @cmds. 562 */ 563 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 564 unsigned long *cmds) 565 { 566 down_write(&cxl_memdev_rwsem); 567 bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 568 CXL_MEM_COMMAND_ID_MAX); 569 up_write(&cxl_memdev_rwsem); 570 } 571 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL); 572 573 /** 574 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 575 * @mds: The device state to modify 576 * @cmds: bitmap of commands to mark available for userspace 577 */ 578 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 579 unsigned long *cmds) 580 { 581 down_write(&cxl_memdev_rwsem); 582 bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 583 CXL_MEM_COMMAND_ID_MAX); 584 up_write(&cxl_memdev_rwsem); 585 } 586 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL); 587 588 static void cxl_memdev_shutdown(struct device *dev) 589 { 590 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 591 592 down_write(&cxl_memdev_rwsem); 593 cxlmd->cxlds = NULL; 594 up_write(&cxl_memdev_rwsem); 595 } 596 597 static void cxl_memdev_unregister(void *_cxlmd) 598 { 599 struct cxl_memdev *cxlmd = _cxlmd; 600 struct device *dev = &cxlmd->dev; 601 602 cdev_device_del(&cxlmd->cdev, dev); 603 cxl_memdev_shutdown(dev); 604 put_device(dev); 605 } 606 607 static void detach_memdev(struct work_struct *work) 608 { 609 struct cxl_memdev *cxlmd; 610 611 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 612 device_release_driver(&cxlmd->dev); 613 put_device(&cxlmd->dev); 614 } 615 616 static struct lock_class_key cxl_memdev_key; 617 618 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 619 const struct file_operations *fops) 620 { 621 struct cxl_memdev *cxlmd; 622 struct device *dev; 623 struct cdev *cdev; 624 int rc; 625 626 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 627 if (!cxlmd) 628 return ERR_PTR(-ENOMEM); 629 630 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 631 if (rc < 0) 632 goto err; 633 cxlmd->id = rc; 634 cxlmd->depth = -1; 635 636 dev = &cxlmd->dev; 637 device_initialize(dev); 638 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 639 dev->parent = cxlds->dev; 640 dev->bus = &cxl_bus_type; 641 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 642 dev->type = &cxl_memdev_type; 643 device_set_pm_not_required(dev); 644 INIT_WORK(&cxlmd->detach_work, detach_memdev); 645 646 cdev = &cxlmd->cdev; 647 cdev_init(cdev, fops); 648 return cxlmd; 649 650 err: 651 kfree(cxlmd); 652 return ERR_PTR(rc); 653 } 654 655 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 656 unsigned long arg) 657 { 658 switch (cmd) { 659 case CXL_MEM_QUERY_COMMANDS: 660 return cxl_query_cmd(cxlmd, (void __user *)arg); 661 case CXL_MEM_SEND_COMMAND: 662 return cxl_send_cmd(cxlmd, (void __user *)arg); 663 default: 664 return -ENOTTY; 665 } 666 } 667 668 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 669 unsigned long arg) 670 { 671 struct cxl_memdev *cxlmd = file->private_data; 672 struct cxl_dev_state *cxlds; 673 int rc = -ENXIO; 674 675 down_read(&cxl_memdev_rwsem); 676 cxlds = cxlmd->cxlds; 677 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 678 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 679 up_read(&cxl_memdev_rwsem); 680 681 return rc; 682 } 683 684 static int cxl_memdev_open(struct inode *inode, struct file *file) 685 { 686 struct cxl_memdev *cxlmd = 687 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 688 689 get_device(&cxlmd->dev); 690 file->private_data = cxlmd; 691 692 return 0; 693 } 694 695 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 696 { 697 struct cxl_memdev *cxlmd = 698 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 699 700 put_device(&cxlmd->dev); 701 702 return 0; 703 } 704 705 /** 706 * cxl_mem_get_fw_info - Get Firmware info 707 * @mds: The device data for the operation 708 * 709 * Retrieve firmware info for the device specified. 710 * 711 * Return: 0 if no error: or the result of the mailbox command. 712 * 713 * See CXL-3.0 8.2.9.3.1 Get FW Info 714 */ 715 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 716 { 717 struct cxl_mbox_get_fw_info info; 718 struct cxl_mbox_cmd mbox_cmd; 719 int rc; 720 721 mbox_cmd = (struct cxl_mbox_cmd) { 722 .opcode = CXL_MBOX_OP_GET_FW_INFO, 723 .size_out = sizeof(info), 724 .payload_out = &info, 725 }; 726 727 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 728 if (rc < 0) 729 return rc; 730 731 mds->fw.num_slots = info.num_slots; 732 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 733 info.slot_info); 734 735 return 0; 736 } 737 738 /** 739 * cxl_mem_activate_fw - Activate Firmware 740 * @mds: The device data for the operation 741 * @slot: slot number to activate 742 * 743 * Activate firmware in a given slot for the device specified. 744 * 745 * Return: 0 if no error: or the result of the mailbox command. 746 * 747 * See CXL-3.0 8.2.9.3.3 Activate FW 748 */ 749 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 750 { 751 struct cxl_mbox_activate_fw activate; 752 struct cxl_mbox_cmd mbox_cmd; 753 754 if (slot == 0 || slot > mds->fw.num_slots) 755 return -EINVAL; 756 757 mbox_cmd = (struct cxl_mbox_cmd) { 758 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 759 .size_in = sizeof(activate), 760 .payload_in = &activate, 761 }; 762 763 /* Only offline activation supported for now */ 764 activate.action = CXL_FW_ACTIVATE_OFFLINE; 765 activate.slot = slot; 766 767 return cxl_internal_send_cmd(mds, &mbox_cmd); 768 } 769 770 /** 771 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 772 * @mds: The device data for the operation 773 * 774 * Abort an in-progress firmware transfer for the device specified. 775 * 776 * Return: 0 if no error: or the result of the mailbox command. 777 * 778 * See CXL-3.0 8.2.9.3.2 Transfer FW 779 */ 780 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 781 { 782 struct cxl_mbox_transfer_fw *transfer; 783 struct cxl_mbox_cmd mbox_cmd; 784 int rc; 785 786 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 787 if (!transfer) 788 return -ENOMEM; 789 790 /* Set a 1s poll interval and a total wait time of 30s */ 791 mbox_cmd = (struct cxl_mbox_cmd) { 792 .opcode = CXL_MBOX_OP_TRANSFER_FW, 793 .size_in = sizeof(*transfer), 794 .payload_in = transfer, 795 .poll_interval_ms = 1000, 796 .poll_count = 30, 797 }; 798 799 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 800 801 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 802 kfree(transfer); 803 return rc; 804 } 805 806 static void cxl_fw_cleanup(struct fw_upload *fwl) 807 { 808 struct cxl_memdev_state *mds = fwl->dd_handle; 809 810 mds->fw.next_slot = 0; 811 } 812 813 static int cxl_fw_do_cancel(struct fw_upload *fwl) 814 { 815 struct cxl_memdev_state *mds = fwl->dd_handle; 816 struct cxl_dev_state *cxlds = &mds->cxlds; 817 struct cxl_memdev *cxlmd = cxlds->cxlmd; 818 int rc; 819 820 rc = cxl_mem_abort_fw_xfer(mds); 821 if (rc < 0) 822 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 823 824 return FW_UPLOAD_ERR_CANCELED; 825 } 826 827 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 828 u32 size) 829 { 830 struct cxl_memdev_state *mds = fwl->dd_handle; 831 struct cxl_mbox_transfer_fw *transfer; 832 833 if (!size) 834 return FW_UPLOAD_ERR_INVALID_SIZE; 835 836 mds->fw.oneshot = struct_size(transfer, data, size) < 837 mds->payload_size; 838 839 if (cxl_mem_get_fw_info(mds)) 840 return FW_UPLOAD_ERR_HW_ERROR; 841 842 /* 843 * So far no state has been changed, hence no other cleanup is 844 * necessary. Simply return the cancelled status. 845 */ 846 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 847 return FW_UPLOAD_ERR_CANCELED; 848 849 return FW_UPLOAD_ERR_NONE; 850 } 851 852 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 853 u32 offset, u32 size, u32 *written) 854 { 855 struct cxl_memdev_state *mds = fwl->dd_handle; 856 struct cxl_dev_state *cxlds = &mds->cxlds; 857 struct cxl_memdev *cxlmd = cxlds->cxlmd; 858 struct cxl_mbox_transfer_fw *transfer; 859 struct cxl_mbox_cmd mbox_cmd; 860 u32 cur_size, remaining; 861 size_t size_in; 862 int rc; 863 864 *written = 0; 865 866 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 867 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 868 dev_err(&cxlmd->dev, 869 "misaligned offset for FW transfer slice (%u)\n", 870 offset); 871 return FW_UPLOAD_ERR_RW_ERROR; 872 } 873 874 /* 875 * Pick transfer size based on mds->payload_size @size must bw 128-byte 876 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 877 * sizeof(*transfer) is 128. These constraints imply that @cur_size 878 * will always be 128b aligned. 879 */ 880 cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer)); 881 882 remaining = size - cur_size; 883 size_in = struct_size(transfer, data, cur_size); 884 885 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 886 return cxl_fw_do_cancel(fwl); 887 888 /* 889 * Slot numbers are 1-indexed 890 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 891 * Check for rollover using modulo, and 1-index it by adding 1 892 */ 893 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 894 895 /* Do the transfer via mailbox cmd */ 896 transfer = kzalloc(size_in, GFP_KERNEL); 897 if (!transfer) 898 return FW_UPLOAD_ERR_RW_ERROR; 899 900 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 901 memcpy(transfer->data, data + offset, cur_size); 902 if (mds->fw.oneshot) { 903 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 904 transfer->slot = mds->fw.next_slot; 905 } else { 906 if (offset == 0) { 907 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 908 } else if (remaining == 0) { 909 transfer->action = CXL_FW_TRANSFER_ACTION_END; 910 transfer->slot = mds->fw.next_slot; 911 } else { 912 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 913 } 914 } 915 916 mbox_cmd = (struct cxl_mbox_cmd) { 917 .opcode = CXL_MBOX_OP_TRANSFER_FW, 918 .size_in = size_in, 919 .payload_in = transfer, 920 .poll_interval_ms = 1000, 921 .poll_count = 30, 922 }; 923 924 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 925 if (rc < 0) { 926 rc = FW_UPLOAD_ERR_RW_ERROR; 927 goto out_free; 928 } 929 930 *written = cur_size; 931 932 /* Activate FW if oneshot or if the last slice was written */ 933 if (mds->fw.oneshot || remaining == 0) { 934 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 935 mds->fw.next_slot); 936 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 937 if (rc < 0) { 938 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 939 rc); 940 rc = FW_UPLOAD_ERR_HW_ERROR; 941 goto out_free; 942 } 943 } 944 945 rc = FW_UPLOAD_ERR_NONE; 946 947 out_free: 948 kfree(transfer); 949 return rc; 950 } 951 952 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 953 { 954 struct cxl_memdev_state *mds = fwl->dd_handle; 955 956 /* 957 * cxl_internal_send_cmd() handles background operations synchronously. 958 * No need to wait for completions here - any errors would've been 959 * reported and handled during the ->write() call(s). 960 * Just check if a cancel request was received, and return success. 961 */ 962 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 963 return cxl_fw_do_cancel(fwl); 964 965 return FW_UPLOAD_ERR_NONE; 966 } 967 968 static void cxl_fw_cancel(struct fw_upload *fwl) 969 { 970 struct cxl_memdev_state *mds = fwl->dd_handle; 971 972 set_bit(CXL_FW_CANCEL, mds->fw.state); 973 } 974 975 static const struct fw_upload_ops cxl_memdev_fw_ops = { 976 .prepare = cxl_fw_prepare, 977 .write = cxl_fw_write, 978 .poll_complete = cxl_fw_poll_complete, 979 .cancel = cxl_fw_cancel, 980 .cleanup = cxl_fw_cleanup, 981 }; 982 983 static void cxl_remove_fw_upload(void *fwl) 984 { 985 firmware_upload_unregister(fwl); 986 } 987 988 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 989 { 990 struct cxl_dev_state *cxlds = &mds->cxlds; 991 struct device *dev = &cxlds->cxlmd->dev; 992 struct fw_upload *fwl; 993 994 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds)) 995 return 0; 996 997 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 998 &cxl_memdev_fw_ops, mds); 999 if (IS_ERR(fwl)) 1000 return PTR_ERR(fwl); 1001 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 1002 } 1003 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, CXL); 1004 1005 static const struct file_operations cxl_memdev_fops = { 1006 .owner = THIS_MODULE, 1007 .unlocked_ioctl = cxl_memdev_ioctl, 1008 .open = cxl_memdev_open, 1009 .release = cxl_memdev_release_file, 1010 .compat_ioctl = compat_ptr_ioctl, 1011 .llseek = noop_llseek, 1012 }; 1013 1014 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 1015 struct cxl_dev_state *cxlds) 1016 { 1017 struct cxl_memdev *cxlmd; 1018 struct device *dev; 1019 struct cdev *cdev; 1020 int rc; 1021 1022 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1023 if (IS_ERR(cxlmd)) 1024 return cxlmd; 1025 1026 dev = &cxlmd->dev; 1027 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1028 if (rc) 1029 goto err; 1030 1031 /* 1032 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1033 * needed as this is ordered with cdev_add() publishing the device. 1034 */ 1035 cxlmd->cxlds = cxlds; 1036 cxlds->cxlmd = cxlmd; 1037 1038 cdev = &cxlmd->cdev; 1039 rc = cdev_device_add(cdev, dev); 1040 if (rc) 1041 goto err; 1042 1043 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1044 if (rc) 1045 return ERR_PTR(rc); 1046 return cxlmd; 1047 1048 err: 1049 /* 1050 * The cdev was briefly live, shutdown any ioctl operations that 1051 * saw that state. 1052 */ 1053 cxl_memdev_shutdown(dev); 1054 put_device(dev); 1055 return ERR_PTR(rc); 1056 } 1057 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL); 1058 1059 static void sanitize_teardown_notifier(void *data) 1060 { 1061 struct cxl_memdev_state *mds = data; 1062 struct kernfs_node *state; 1063 1064 /* 1065 * Prevent new irq triggered invocations of the workqueue and 1066 * flush inflight invocations. 1067 */ 1068 mutex_lock(&mds->mbox_mutex); 1069 state = mds->security.sanitize_node; 1070 mds->security.sanitize_node = NULL; 1071 mutex_unlock(&mds->mbox_mutex); 1072 1073 cancel_delayed_work_sync(&mds->security.poll_dwork); 1074 sysfs_put(state); 1075 } 1076 1077 int devm_cxl_sanitize_setup_notifier(struct device *host, 1078 struct cxl_memdev *cxlmd) 1079 { 1080 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1081 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1082 struct kernfs_node *sec; 1083 1084 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1085 return 0; 1086 1087 /* 1088 * Note, the expectation is that @cxlmd would have failed to be 1089 * created if these sysfs_get_dirent calls fail. 1090 */ 1091 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1092 if (!sec) 1093 return -ENOENT; 1094 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1095 sysfs_put(sec); 1096 if (!mds->security.sanitize_node) 1097 return -ENOENT; 1098 1099 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1100 } 1101 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, CXL); 1102 1103 __init int cxl_memdev_init(void) 1104 { 1105 dev_t devt; 1106 int rc; 1107 1108 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1109 if (rc) 1110 return rc; 1111 1112 cxl_mem_major = MAJOR(devt); 1113 1114 return 0; 1115 } 1116 1117 void cxl_memdev_exit(void) 1118 { 1119 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1120 } 1121