1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 kfree(cxlmd); 31 } 32 33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 34 kgid_t *gid) 35 { 36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 37 } 38 39 static ssize_t firmware_version_show(struct device *dev, 40 struct device_attribute *attr, char *buf) 41 { 42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 43 struct cxl_dev_state *cxlds = cxlmd->cxlds; 44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 45 46 if (!mds) 47 return sysfs_emit(buf, "\n"); 48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 49 } 50 static DEVICE_ATTR_RO(firmware_version); 51 52 static ssize_t payload_max_show(struct device *dev, 53 struct device_attribute *attr, char *buf) 54 { 55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 56 struct cxl_dev_state *cxlds = cxlmd->cxlds; 57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 58 59 if (!mds) 60 return sysfs_emit(buf, "\n"); 61 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size); 62 } 63 static DEVICE_ATTR_RO(payload_max); 64 65 static ssize_t label_storage_size_show(struct device *dev, 66 struct device_attribute *attr, char *buf) 67 { 68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 69 struct cxl_dev_state *cxlds = cxlmd->cxlds; 70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 71 72 if (!mds) 73 return sysfs_emit(buf, "\n"); 74 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 75 } 76 static DEVICE_ATTR_RO(label_storage_size); 77 78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 79 char *buf) 80 { 81 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 82 struct cxl_dev_state *cxlds = cxlmd->cxlds; 83 unsigned long long len = resource_size(&cxlds->ram_res); 84 85 return sysfs_emit(buf, "%#llx\n", len); 86 } 87 88 static struct device_attribute dev_attr_ram_size = 89 __ATTR(size, 0444, ram_size_show, NULL); 90 91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 92 char *buf) 93 { 94 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 95 struct cxl_dev_state *cxlds = cxlmd->cxlds; 96 unsigned long long len = resource_size(&cxlds->pmem_res); 97 98 return sysfs_emit(buf, "%#llx\n", len); 99 } 100 101 static struct device_attribute dev_attr_pmem_size = 102 __ATTR(size, 0444, pmem_size_show, NULL); 103 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 105 char *buf) 106 { 107 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 108 struct cxl_dev_state *cxlds = cxlmd->cxlds; 109 110 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 111 } 112 static DEVICE_ATTR_RO(serial); 113 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 115 char *buf) 116 { 117 return sysfs_emit(buf, "%d\n", dev_to_node(dev)); 118 } 119 static DEVICE_ATTR_RO(numa_node); 120 121 static ssize_t security_state_show(struct device *dev, 122 struct device_attribute *attr, 123 char *buf) 124 { 125 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 126 struct cxl_dev_state *cxlds = cxlmd->cxlds; 127 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 128 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 129 unsigned long state = mds->security.state; 130 int rc = 0; 131 132 /* sync with latest submission state */ 133 mutex_lock(&cxl_mbox->mbox_mutex); 134 if (mds->security.sanitize_active) 135 rc = sysfs_emit(buf, "sanitize\n"); 136 mutex_unlock(&cxl_mbox->mbox_mutex); 137 if (rc) 138 return rc; 139 140 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 141 return sysfs_emit(buf, "disabled\n"); 142 if (state & CXL_PMEM_SEC_STATE_FROZEN || 143 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 144 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 145 return sysfs_emit(buf, "frozen\n"); 146 if (state & CXL_PMEM_SEC_STATE_LOCKED) 147 return sysfs_emit(buf, "locked\n"); 148 else 149 return sysfs_emit(buf, "unlocked\n"); 150 } 151 static struct device_attribute dev_attr_security_state = 152 __ATTR(state, 0444, security_state_show, NULL); 153 154 static ssize_t security_sanitize_store(struct device *dev, 155 struct device_attribute *attr, 156 const char *buf, size_t len) 157 { 158 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 159 bool sanitize; 160 ssize_t rc; 161 162 if (kstrtobool(buf, &sanitize) || !sanitize) 163 return -EINVAL; 164 165 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 166 if (rc) 167 return rc; 168 169 return len; 170 } 171 static struct device_attribute dev_attr_security_sanitize = 172 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 173 174 static ssize_t security_erase_store(struct device *dev, 175 struct device_attribute *attr, 176 const char *buf, size_t len) 177 { 178 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 179 ssize_t rc; 180 bool erase; 181 182 if (kstrtobool(buf, &erase) || !erase) 183 return -EINVAL; 184 185 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 186 if (rc) 187 return rc; 188 189 return len; 190 } 191 static struct device_attribute dev_attr_security_erase = 192 __ATTR(erase, 0200, NULL, security_erase_store); 193 194 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 195 { 196 struct cxl_dev_state *cxlds = cxlmd->cxlds; 197 u64 offset, length; 198 int rc = 0; 199 200 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 201 if (resource_size(&cxlds->pmem_res)) { 202 offset = cxlds->pmem_res.start; 203 length = resource_size(&cxlds->pmem_res); 204 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 205 if (rc) 206 return rc; 207 } 208 if (resource_size(&cxlds->ram_res)) { 209 offset = cxlds->ram_res.start; 210 length = resource_size(&cxlds->ram_res); 211 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 212 /* 213 * Invalid Physical Address is not an error for 214 * volatile addresses. Device support is optional. 215 */ 216 if (rc == -EFAULT) 217 rc = 0; 218 } 219 return rc; 220 } 221 222 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 223 { 224 struct cxl_port *port; 225 int rc; 226 227 port = cxlmd->endpoint; 228 if (!port || !is_cxl_endpoint(port)) 229 return -EINVAL; 230 231 rc = down_read_interruptible(&cxl_region_rwsem); 232 if (rc) 233 return rc; 234 235 rc = down_read_interruptible(&cxl_dpa_rwsem); 236 if (rc) { 237 up_read(&cxl_region_rwsem); 238 return rc; 239 } 240 241 if (cxl_num_decoders_committed(port) == 0) { 242 /* No regions mapped to this memdev */ 243 rc = cxl_get_poison_by_memdev(cxlmd); 244 } else { 245 /* Regions mapped, collect poison by endpoint */ 246 rc = cxl_get_poison_by_endpoint(port); 247 } 248 up_read(&cxl_dpa_rwsem); 249 up_read(&cxl_region_rwsem); 250 251 return rc; 252 } 253 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL"); 254 255 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 256 { 257 struct cxl_dev_state *cxlds = cxlmd->cxlds; 258 259 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 260 return 0; 261 262 if (!resource_size(&cxlds->dpa_res)) { 263 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 264 return -EINVAL; 265 } 266 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 267 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 268 dpa, &cxlds->dpa_res); 269 return -EINVAL; 270 } 271 if (!IS_ALIGNED(dpa, 64)) { 272 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 273 return -EINVAL; 274 } 275 276 return 0; 277 } 278 279 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 280 { 281 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 282 struct cxl_mbox_inject_poison inject; 283 struct cxl_poison_record record; 284 struct cxl_mbox_cmd mbox_cmd; 285 struct cxl_region *cxlr; 286 int rc; 287 288 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 289 return 0; 290 291 rc = down_read_interruptible(&cxl_region_rwsem); 292 if (rc) 293 return rc; 294 295 rc = down_read_interruptible(&cxl_dpa_rwsem); 296 if (rc) { 297 up_read(&cxl_region_rwsem); 298 return rc; 299 } 300 301 rc = cxl_validate_poison_dpa(cxlmd, dpa); 302 if (rc) 303 goto out; 304 305 inject.address = cpu_to_le64(dpa); 306 mbox_cmd = (struct cxl_mbox_cmd) { 307 .opcode = CXL_MBOX_OP_INJECT_POISON, 308 .size_in = sizeof(inject), 309 .payload_in = &inject, 310 }; 311 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 312 if (rc) 313 goto out; 314 315 cxlr = cxl_dpa_to_region(cxlmd, dpa); 316 if (cxlr) 317 dev_warn_once(cxl_mbox->host, 318 "poison inject dpa:%#llx region: %s\n", dpa, 319 dev_name(&cxlr->dev)); 320 321 record = (struct cxl_poison_record) { 322 .address = cpu_to_le64(dpa), 323 .length = cpu_to_le32(1), 324 }; 325 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 326 out: 327 up_read(&cxl_dpa_rwsem); 328 up_read(&cxl_region_rwsem); 329 330 return rc; 331 } 332 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL"); 333 334 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 335 { 336 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 337 struct cxl_mbox_clear_poison clear; 338 struct cxl_poison_record record; 339 struct cxl_mbox_cmd mbox_cmd; 340 struct cxl_region *cxlr; 341 int rc; 342 343 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 344 return 0; 345 346 rc = down_read_interruptible(&cxl_region_rwsem); 347 if (rc) 348 return rc; 349 350 rc = down_read_interruptible(&cxl_dpa_rwsem); 351 if (rc) { 352 up_read(&cxl_region_rwsem); 353 return rc; 354 } 355 356 rc = cxl_validate_poison_dpa(cxlmd, dpa); 357 if (rc) 358 goto out; 359 360 /* 361 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 362 * is defined to accept 64 bytes of write-data, along with the 363 * address to clear. This driver uses zeroes as write-data. 364 */ 365 clear = (struct cxl_mbox_clear_poison) { 366 .address = cpu_to_le64(dpa) 367 }; 368 369 mbox_cmd = (struct cxl_mbox_cmd) { 370 .opcode = CXL_MBOX_OP_CLEAR_POISON, 371 .size_in = sizeof(clear), 372 .payload_in = &clear, 373 }; 374 375 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 376 if (rc) 377 goto out; 378 379 cxlr = cxl_dpa_to_region(cxlmd, dpa); 380 if (cxlr) 381 dev_warn_once(cxl_mbox->host, 382 "poison clear dpa:%#llx region: %s\n", dpa, 383 dev_name(&cxlr->dev)); 384 385 record = (struct cxl_poison_record) { 386 .address = cpu_to_le64(dpa), 387 .length = cpu_to_le32(1), 388 }; 389 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 390 out: 391 up_read(&cxl_dpa_rwsem); 392 up_read(&cxl_region_rwsem); 393 394 return rc; 395 } 396 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL"); 397 398 static struct attribute *cxl_memdev_attributes[] = { 399 &dev_attr_serial.attr, 400 &dev_attr_firmware_version.attr, 401 &dev_attr_payload_max.attr, 402 &dev_attr_label_storage_size.attr, 403 &dev_attr_numa_node.attr, 404 NULL, 405 }; 406 407 static ssize_t pmem_qos_class_show(struct device *dev, 408 struct device_attribute *attr, char *buf) 409 { 410 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 411 struct cxl_dev_state *cxlds = cxlmd->cxlds; 412 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 413 414 return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class); 415 } 416 417 static struct device_attribute dev_attr_pmem_qos_class = 418 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); 419 420 static struct attribute *cxl_memdev_pmem_attributes[] = { 421 &dev_attr_pmem_size.attr, 422 &dev_attr_pmem_qos_class.attr, 423 NULL, 424 }; 425 426 static ssize_t ram_qos_class_show(struct device *dev, 427 struct device_attribute *attr, char *buf) 428 { 429 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 430 struct cxl_dev_state *cxlds = cxlmd->cxlds; 431 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 432 433 return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class); 434 } 435 436 static struct device_attribute dev_attr_ram_qos_class = 437 __ATTR(qos_class, 0444, ram_qos_class_show, NULL); 438 439 static struct attribute *cxl_memdev_ram_attributes[] = { 440 &dev_attr_ram_size.attr, 441 &dev_attr_ram_qos_class.attr, 442 NULL, 443 }; 444 445 static struct attribute *cxl_memdev_security_attributes[] = { 446 &dev_attr_security_state.attr, 447 &dev_attr_security_sanitize.attr, 448 &dev_attr_security_erase.attr, 449 NULL, 450 }; 451 452 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 453 int n) 454 { 455 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 456 return 0; 457 return a->mode; 458 } 459 460 static struct attribute_group cxl_memdev_attribute_group = { 461 .attrs = cxl_memdev_attributes, 462 .is_visible = cxl_memdev_visible, 463 }; 464 465 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) 466 { 467 struct device *dev = kobj_to_dev(kobj); 468 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 469 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 470 471 if (a == &dev_attr_ram_qos_class.attr) 472 if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID) 473 return 0; 474 475 return a->mode; 476 } 477 478 static struct attribute_group cxl_memdev_ram_attribute_group = { 479 .name = "ram", 480 .attrs = cxl_memdev_ram_attributes, 481 .is_visible = cxl_ram_visible, 482 }; 483 484 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) 485 { 486 struct device *dev = kobj_to_dev(kobj); 487 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 488 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 489 490 if (a == &dev_attr_pmem_qos_class.attr) 491 if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID) 492 return 0; 493 494 return a->mode; 495 } 496 497 static struct attribute_group cxl_memdev_pmem_attribute_group = { 498 .name = "pmem", 499 .attrs = cxl_memdev_pmem_attributes, 500 .is_visible = cxl_pmem_visible, 501 }; 502 503 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 504 struct attribute *a, int n) 505 { 506 struct device *dev = kobj_to_dev(kobj); 507 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 508 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 509 510 if (a == &dev_attr_security_sanitize.attr && 511 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 512 return 0; 513 514 if (a == &dev_attr_security_erase.attr && 515 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 516 return 0; 517 518 return a->mode; 519 } 520 521 static struct attribute_group cxl_memdev_security_attribute_group = { 522 .name = "security", 523 .attrs = cxl_memdev_security_attributes, 524 .is_visible = cxl_memdev_security_visible, 525 }; 526 527 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 528 &cxl_memdev_attribute_group, 529 &cxl_memdev_ram_attribute_group, 530 &cxl_memdev_pmem_attribute_group, 531 &cxl_memdev_security_attribute_group, 532 NULL, 533 }; 534 535 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) 536 { 537 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); 538 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); 539 } 540 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL"); 541 542 static const struct device_type cxl_memdev_type = { 543 .name = "cxl_memdev", 544 .release = cxl_memdev_release, 545 .devnode = cxl_memdev_devnode, 546 .groups = cxl_memdev_attribute_groups, 547 }; 548 549 bool is_cxl_memdev(const struct device *dev) 550 { 551 return dev->type == &cxl_memdev_type; 552 } 553 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); 554 555 /** 556 * set_exclusive_cxl_commands() - atomically disable user cxl commands 557 * @mds: The device state to operate on 558 * @cmds: bitmap of commands to mark exclusive 559 * 560 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 561 * invocations of the ioctl path and then disable future execution of 562 * commands with the command ids set in @cmds. 563 */ 564 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 565 unsigned long *cmds) 566 { 567 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 568 569 down_write(&cxl_memdev_rwsem); 570 bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 571 cmds, CXL_MEM_COMMAND_ID_MAX); 572 up_write(&cxl_memdev_rwsem); 573 } 574 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL"); 575 576 /** 577 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 578 * @mds: The device state to modify 579 * @cmds: bitmap of commands to mark available for userspace 580 */ 581 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 582 unsigned long *cmds) 583 { 584 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 585 586 down_write(&cxl_memdev_rwsem); 587 bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 588 cmds, CXL_MEM_COMMAND_ID_MAX); 589 up_write(&cxl_memdev_rwsem); 590 } 591 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL"); 592 593 static void cxl_memdev_shutdown(struct device *dev) 594 { 595 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 596 597 down_write(&cxl_memdev_rwsem); 598 cxlmd->cxlds = NULL; 599 up_write(&cxl_memdev_rwsem); 600 } 601 602 static void cxl_memdev_unregister(void *_cxlmd) 603 { 604 struct cxl_memdev *cxlmd = _cxlmd; 605 struct device *dev = &cxlmd->dev; 606 607 cdev_device_del(&cxlmd->cdev, dev); 608 cxl_memdev_shutdown(dev); 609 put_device(dev); 610 } 611 612 static void detach_memdev(struct work_struct *work) 613 { 614 struct cxl_memdev *cxlmd; 615 616 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 617 device_release_driver(&cxlmd->dev); 618 put_device(&cxlmd->dev); 619 } 620 621 static struct lock_class_key cxl_memdev_key; 622 623 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 624 const struct file_operations *fops) 625 { 626 struct cxl_memdev *cxlmd; 627 struct device *dev; 628 struct cdev *cdev; 629 int rc; 630 631 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 632 if (!cxlmd) 633 return ERR_PTR(-ENOMEM); 634 635 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 636 if (rc < 0) 637 goto err; 638 cxlmd->id = rc; 639 cxlmd->depth = -1; 640 641 dev = &cxlmd->dev; 642 device_initialize(dev); 643 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 644 dev->parent = cxlds->dev; 645 dev->bus = &cxl_bus_type; 646 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 647 dev->type = &cxl_memdev_type; 648 device_set_pm_not_required(dev); 649 INIT_WORK(&cxlmd->detach_work, detach_memdev); 650 651 cdev = &cxlmd->cdev; 652 cdev_init(cdev, fops); 653 return cxlmd; 654 655 err: 656 kfree(cxlmd); 657 return ERR_PTR(rc); 658 } 659 660 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 661 unsigned long arg) 662 { 663 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 664 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 665 666 switch (cmd) { 667 case CXL_MEM_QUERY_COMMANDS: 668 return cxl_query_cmd(cxl_mbox, (void __user *)arg); 669 case CXL_MEM_SEND_COMMAND: 670 return cxl_send_cmd(cxl_mbox, (void __user *)arg); 671 default: 672 return -ENOTTY; 673 } 674 } 675 676 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 677 unsigned long arg) 678 { 679 struct cxl_memdev *cxlmd = file->private_data; 680 struct cxl_dev_state *cxlds; 681 int rc = -ENXIO; 682 683 down_read(&cxl_memdev_rwsem); 684 cxlds = cxlmd->cxlds; 685 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 686 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 687 up_read(&cxl_memdev_rwsem); 688 689 return rc; 690 } 691 692 static int cxl_memdev_open(struct inode *inode, struct file *file) 693 { 694 struct cxl_memdev *cxlmd = 695 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 696 697 get_device(&cxlmd->dev); 698 file->private_data = cxlmd; 699 700 return 0; 701 } 702 703 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 704 { 705 struct cxl_memdev *cxlmd = 706 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 707 708 put_device(&cxlmd->dev); 709 710 return 0; 711 } 712 713 /** 714 * cxl_mem_get_fw_info - Get Firmware info 715 * @mds: The device data for the operation 716 * 717 * Retrieve firmware info for the device specified. 718 * 719 * Return: 0 if no error: or the result of the mailbox command. 720 * 721 * See CXL-3.0 8.2.9.3.1 Get FW Info 722 */ 723 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 724 { 725 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 726 struct cxl_mbox_get_fw_info info; 727 struct cxl_mbox_cmd mbox_cmd; 728 int rc; 729 730 mbox_cmd = (struct cxl_mbox_cmd) { 731 .opcode = CXL_MBOX_OP_GET_FW_INFO, 732 .size_out = sizeof(info), 733 .payload_out = &info, 734 }; 735 736 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 737 if (rc < 0) 738 return rc; 739 740 mds->fw.num_slots = info.num_slots; 741 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 742 info.slot_info); 743 744 return 0; 745 } 746 747 /** 748 * cxl_mem_activate_fw - Activate Firmware 749 * @mds: The device data for the operation 750 * @slot: slot number to activate 751 * 752 * Activate firmware in a given slot for the device specified. 753 * 754 * Return: 0 if no error: or the result of the mailbox command. 755 * 756 * See CXL-3.0 8.2.9.3.3 Activate FW 757 */ 758 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 759 { 760 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 761 struct cxl_mbox_activate_fw activate; 762 struct cxl_mbox_cmd mbox_cmd; 763 764 if (slot == 0 || slot > mds->fw.num_slots) 765 return -EINVAL; 766 767 mbox_cmd = (struct cxl_mbox_cmd) { 768 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 769 .size_in = sizeof(activate), 770 .payload_in = &activate, 771 }; 772 773 /* Only offline activation supported for now */ 774 activate.action = CXL_FW_ACTIVATE_OFFLINE; 775 activate.slot = slot; 776 777 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 778 } 779 780 /** 781 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 782 * @mds: The device data for the operation 783 * 784 * Abort an in-progress firmware transfer for the device specified. 785 * 786 * Return: 0 if no error: or the result of the mailbox command. 787 * 788 * See CXL-3.0 8.2.9.3.2 Transfer FW 789 */ 790 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 791 { 792 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 793 struct cxl_mbox_transfer_fw *transfer; 794 struct cxl_mbox_cmd mbox_cmd; 795 int rc; 796 797 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 798 if (!transfer) 799 return -ENOMEM; 800 801 /* Set a 1s poll interval and a total wait time of 30s */ 802 mbox_cmd = (struct cxl_mbox_cmd) { 803 .opcode = CXL_MBOX_OP_TRANSFER_FW, 804 .size_in = sizeof(*transfer), 805 .payload_in = transfer, 806 .poll_interval_ms = 1000, 807 .poll_count = 30, 808 }; 809 810 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 811 812 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 813 kfree(transfer); 814 return rc; 815 } 816 817 static void cxl_fw_cleanup(struct fw_upload *fwl) 818 { 819 struct cxl_memdev_state *mds = fwl->dd_handle; 820 821 mds->fw.next_slot = 0; 822 } 823 824 static int cxl_fw_do_cancel(struct fw_upload *fwl) 825 { 826 struct cxl_memdev_state *mds = fwl->dd_handle; 827 struct cxl_dev_state *cxlds = &mds->cxlds; 828 struct cxl_memdev *cxlmd = cxlds->cxlmd; 829 int rc; 830 831 rc = cxl_mem_abort_fw_xfer(mds); 832 if (rc < 0) 833 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 834 835 return FW_UPLOAD_ERR_CANCELED; 836 } 837 838 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 839 u32 size) 840 { 841 struct cxl_memdev_state *mds = fwl->dd_handle; 842 struct cxl_mbox_transfer_fw *transfer; 843 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 844 845 if (!size) 846 return FW_UPLOAD_ERR_INVALID_SIZE; 847 848 mds->fw.oneshot = struct_size(transfer, data, size) < 849 cxl_mbox->payload_size; 850 851 if (cxl_mem_get_fw_info(mds)) 852 return FW_UPLOAD_ERR_HW_ERROR; 853 854 /* 855 * So far no state has been changed, hence no other cleanup is 856 * necessary. Simply return the cancelled status. 857 */ 858 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 859 return FW_UPLOAD_ERR_CANCELED; 860 861 return FW_UPLOAD_ERR_NONE; 862 } 863 864 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 865 u32 offset, u32 size, u32 *written) 866 { 867 struct cxl_memdev_state *mds = fwl->dd_handle; 868 struct cxl_dev_state *cxlds = &mds->cxlds; 869 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 870 struct cxl_memdev *cxlmd = cxlds->cxlmd; 871 struct cxl_mbox_transfer_fw *transfer; 872 struct cxl_mbox_cmd mbox_cmd; 873 u32 cur_size, remaining; 874 size_t size_in; 875 int rc; 876 877 *written = 0; 878 879 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 880 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 881 dev_err(&cxlmd->dev, 882 "misaligned offset for FW transfer slice (%u)\n", 883 offset); 884 return FW_UPLOAD_ERR_RW_ERROR; 885 } 886 887 /* 888 * Pick transfer size based on mds->payload_size @size must bw 128-byte 889 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 890 * sizeof(*transfer) is 128. These constraints imply that @cur_size 891 * will always be 128b aligned. 892 */ 893 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer)); 894 895 remaining = size - cur_size; 896 size_in = struct_size(transfer, data, cur_size); 897 898 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 899 return cxl_fw_do_cancel(fwl); 900 901 /* 902 * Slot numbers are 1-indexed 903 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 904 * Check for rollover using modulo, and 1-index it by adding 1 905 */ 906 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 907 908 /* Do the transfer via mailbox cmd */ 909 transfer = kzalloc(size_in, GFP_KERNEL); 910 if (!transfer) 911 return FW_UPLOAD_ERR_RW_ERROR; 912 913 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 914 memcpy(transfer->data, data + offset, cur_size); 915 if (mds->fw.oneshot) { 916 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 917 transfer->slot = mds->fw.next_slot; 918 } else { 919 if (offset == 0) { 920 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 921 } else if (remaining == 0) { 922 transfer->action = CXL_FW_TRANSFER_ACTION_END; 923 transfer->slot = mds->fw.next_slot; 924 } else { 925 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 926 } 927 } 928 929 mbox_cmd = (struct cxl_mbox_cmd) { 930 .opcode = CXL_MBOX_OP_TRANSFER_FW, 931 .size_in = size_in, 932 .payload_in = transfer, 933 .poll_interval_ms = 1000, 934 .poll_count = 30, 935 }; 936 937 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 938 if (rc < 0) { 939 rc = FW_UPLOAD_ERR_RW_ERROR; 940 goto out_free; 941 } 942 943 *written = cur_size; 944 945 /* Activate FW if oneshot or if the last slice was written */ 946 if (mds->fw.oneshot || remaining == 0) { 947 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 948 mds->fw.next_slot); 949 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 950 if (rc < 0) { 951 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 952 rc); 953 rc = FW_UPLOAD_ERR_HW_ERROR; 954 goto out_free; 955 } 956 } 957 958 rc = FW_UPLOAD_ERR_NONE; 959 960 out_free: 961 kfree(transfer); 962 return rc; 963 } 964 965 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 966 { 967 struct cxl_memdev_state *mds = fwl->dd_handle; 968 969 /* 970 * cxl_internal_send_cmd() handles background operations synchronously. 971 * No need to wait for completions here - any errors would've been 972 * reported and handled during the ->write() call(s). 973 * Just check if a cancel request was received, and return success. 974 */ 975 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 976 return cxl_fw_do_cancel(fwl); 977 978 return FW_UPLOAD_ERR_NONE; 979 } 980 981 static void cxl_fw_cancel(struct fw_upload *fwl) 982 { 983 struct cxl_memdev_state *mds = fwl->dd_handle; 984 985 set_bit(CXL_FW_CANCEL, mds->fw.state); 986 } 987 988 static const struct fw_upload_ops cxl_memdev_fw_ops = { 989 .prepare = cxl_fw_prepare, 990 .write = cxl_fw_write, 991 .poll_complete = cxl_fw_poll_complete, 992 .cancel = cxl_fw_cancel, 993 .cleanup = cxl_fw_cleanup, 994 }; 995 996 static void cxl_remove_fw_upload(void *fwl) 997 { 998 firmware_upload_unregister(fwl); 999 } 1000 1001 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 1002 { 1003 struct cxl_dev_state *cxlds = &mds->cxlds; 1004 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 1005 struct device *dev = &cxlds->cxlmd->dev; 1006 struct fw_upload *fwl; 1007 1008 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds)) 1009 return 0; 1010 1011 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 1012 &cxl_memdev_fw_ops, mds); 1013 if (IS_ERR(fwl)) 1014 return PTR_ERR(fwl); 1015 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 1016 } 1017 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL"); 1018 1019 static const struct file_operations cxl_memdev_fops = { 1020 .owner = THIS_MODULE, 1021 .unlocked_ioctl = cxl_memdev_ioctl, 1022 .open = cxl_memdev_open, 1023 .release = cxl_memdev_release_file, 1024 .compat_ioctl = compat_ptr_ioctl, 1025 .llseek = noop_llseek, 1026 }; 1027 1028 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 1029 struct cxl_dev_state *cxlds) 1030 { 1031 struct cxl_memdev *cxlmd; 1032 struct device *dev; 1033 struct cdev *cdev; 1034 int rc; 1035 1036 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1037 if (IS_ERR(cxlmd)) 1038 return cxlmd; 1039 1040 dev = &cxlmd->dev; 1041 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1042 if (rc) 1043 goto err; 1044 1045 /* 1046 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1047 * needed as this is ordered with cdev_add() publishing the device. 1048 */ 1049 cxlmd->cxlds = cxlds; 1050 cxlds->cxlmd = cxlmd; 1051 1052 cdev = &cxlmd->cdev; 1053 rc = cdev_device_add(cdev, dev); 1054 if (rc) 1055 goto err; 1056 1057 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1058 if (rc) 1059 return ERR_PTR(rc); 1060 return cxlmd; 1061 1062 err: 1063 /* 1064 * The cdev was briefly live, shutdown any ioctl operations that 1065 * saw that state. 1066 */ 1067 cxl_memdev_shutdown(dev); 1068 put_device(dev); 1069 return ERR_PTR(rc); 1070 } 1071 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL"); 1072 1073 static void sanitize_teardown_notifier(void *data) 1074 { 1075 struct cxl_memdev_state *mds = data; 1076 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 1077 struct kernfs_node *state; 1078 1079 /* 1080 * Prevent new irq triggered invocations of the workqueue and 1081 * flush inflight invocations. 1082 */ 1083 mutex_lock(&cxl_mbox->mbox_mutex); 1084 state = mds->security.sanitize_node; 1085 mds->security.sanitize_node = NULL; 1086 mutex_unlock(&cxl_mbox->mbox_mutex); 1087 1088 cancel_delayed_work_sync(&mds->security.poll_dwork); 1089 sysfs_put(state); 1090 } 1091 1092 int devm_cxl_sanitize_setup_notifier(struct device *host, 1093 struct cxl_memdev *cxlmd) 1094 { 1095 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1096 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1097 struct kernfs_node *sec; 1098 1099 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1100 return 0; 1101 1102 /* 1103 * Note, the expectation is that @cxlmd would have failed to be 1104 * created if these sysfs_get_dirent calls fail. 1105 */ 1106 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1107 if (!sec) 1108 return -ENOENT; 1109 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1110 sysfs_put(sec); 1111 if (!mds->security.sanitize_node) 1112 return -ENOENT; 1113 1114 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1115 } 1116 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL"); 1117 1118 __init int cxl_memdev_init(void) 1119 { 1120 dev_t devt; 1121 int rc; 1122 1123 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1124 if (rc) 1125 return rc; 1126 1127 cxl_mem_major = MAJOR(devt); 1128 1129 return 0; 1130 } 1131 1132 void cxl_memdev_exit(void) 1133 { 1134 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1135 } 1136