1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 kfree(cxlmd); 31 } 32 33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 34 kgid_t *gid) 35 { 36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 37 } 38 39 static ssize_t firmware_version_show(struct device *dev, 40 struct device_attribute *attr, char *buf) 41 { 42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 43 struct cxl_dev_state *cxlds = cxlmd->cxlds; 44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 45 46 if (!mds) 47 return sysfs_emit(buf, "\n"); 48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 49 } 50 static DEVICE_ATTR_RO(firmware_version); 51 52 static ssize_t payload_max_show(struct device *dev, 53 struct device_attribute *attr, char *buf) 54 { 55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 56 struct cxl_dev_state *cxlds = cxlmd->cxlds; 57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 58 59 if (!mds) 60 return sysfs_emit(buf, "\n"); 61 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size); 62 } 63 static DEVICE_ATTR_RO(payload_max); 64 65 static ssize_t label_storage_size_show(struct device *dev, 66 struct device_attribute *attr, char *buf) 67 { 68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 69 struct cxl_dev_state *cxlds = cxlmd->cxlds; 70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 71 72 if (!mds) 73 return sysfs_emit(buf, "\n"); 74 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 75 } 76 static DEVICE_ATTR_RO(label_storage_size); 77 78 static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds) 79 { 80 /* Static RAM is only expected at partition 0. */ 81 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 82 return 0; 83 return resource_size(&cxlds->part[0].res); 84 } 85 86 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 87 char *buf) 88 { 89 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 90 struct cxl_dev_state *cxlds = cxlmd->cxlds; 91 unsigned long long len = cxl_ram_size(cxlds); 92 93 return sysfs_emit(buf, "%#llx\n", len); 94 } 95 96 static struct device_attribute dev_attr_ram_size = 97 __ATTR(size, 0444, ram_size_show, NULL); 98 99 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 100 char *buf) 101 { 102 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 103 struct cxl_dev_state *cxlds = cxlmd->cxlds; 104 unsigned long long len = cxl_pmem_size(cxlds); 105 106 return sysfs_emit(buf, "%#llx\n", len); 107 } 108 109 static struct device_attribute dev_attr_pmem_size = 110 __ATTR(size, 0444, pmem_size_show, NULL); 111 112 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 113 char *buf) 114 { 115 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 116 struct cxl_dev_state *cxlds = cxlmd->cxlds; 117 118 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 119 } 120 static DEVICE_ATTR_RO(serial); 121 122 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 123 char *buf) 124 { 125 return sysfs_emit(buf, "%d\n", dev_to_node(dev)); 126 } 127 static DEVICE_ATTR_RO(numa_node); 128 129 static ssize_t security_state_show(struct device *dev, 130 struct device_attribute *attr, 131 char *buf) 132 { 133 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 134 struct cxl_dev_state *cxlds = cxlmd->cxlds; 135 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 136 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 137 unsigned long state = mds->security.state; 138 int rc = 0; 139 140 /* sync with latest submission state */ 141 mutex_lock(&cxl_mbox->mbox_mutex); 142 if (mds->security.sanitize_active) 143 rc = sysfs_emit(buf, "sanitize\n"); 144 mutex_unlock(&cxl_mbox->mbox_mutex); 145 if (rc) 146 return rc; 147 148 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 149 return sysfs_emit(buf, "disabled\n"); 150 if (state & CXL_PMEM_SEC_STATE_FROZEN || 151 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 152 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 153 return sysfs_emit(buf, "frozen\n"); 154 if (state & CXL_PMEM_SEC_STATE_LOCKED) 155 return sysfs_emit(buf, "locked\n"); 156 else 157 return sysfs_emit(buf, "unlocked\n"); 158 } 159 static struct device_attribute dev_attr_security_state = 160 __ATTR(state, 0444, security_state_show, NULL); 161 162 static ssize_t security_sanitize_store(struct device *dev, 163 struct device_attribute *attr, 164 const char *buf, size_t len) 165 { 166 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 167 bool sanitize; 168 ssize_t rc; 169 170 if (kstrtobool(buf, &sanitize) || !sanitize) 171 return -EINVAL; 172 173 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 174 if (rc) 175 return rc; 176 177 return len; 178 } 179 static struct device_attribute dev_attr_security_sanitize = 180 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 181 182 static ssize_t security_erase_store(struct device *dev, 183 struct device_attribute *attr, 184 const char *buf, size_t len) 185 { 186 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 187 ssize_t rc; 188 bool erase; 189 190 if (kstrtobool(buf, &erase) || !erase) 191 return -EINVAL; 192 193 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 194 if (rc) 195 return rc; 196 197 return len; 198 } 199 static struct device_attribute dev_attr_security_erase = 200 __ATTR(erase, 0200, NULL, security_erase_store); 201 202 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 203 { 204 struct cxl_dev_state *cxlds = cxlmd->cxlds; 205 u64 offset, length; 206 int rc = 0; 207 208 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 209 for (int i = 0; i < cxlds->nr_partitions; i++) { 210 const struct resource *res = &cxlds->part[i].res; 211 212 offset = res->start; 213 length = resource_size(res); 214 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 215 /* 216 * Invalid Physical Address is not an error for 217 * volatile addresses. Device support is optional. 218 */ 219 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) 220 rc = 0; 221 } 222 return rc; 223 } 224 225 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 226 { 227 struct cxl_port *port; 228 int rc; 229 230 port = cxlmd->endpoint; 231 if (!port || !is_cxl_endpoint(port)) 232 return -EINVAL; 233 234 rc = down_read_interruptible(&cxl_region_rwsem); 235 if (rc) 236 return rc; 237 238 rc = down_read_interruptible(&cxl_dpa_rwsem); 239 if (rc) { 240 up_read(&cxl_region_rwsem); 241 return rc; 242 } 243 244 if (cxl_num_decoders_committed(port) == 0) { 245 /* No regions mapped to this memdev */ 246 rc = cxl_get_poison_by_memdev(cxlmd); 247 } else { 248 /* Regions mapped, collect poison by endpoint */ 249 rc = cxl_get_poison_by_endpoint(port); 250 } 251 up_read(&cxl_dpa_rwsem); 252 up_read(&cxl_region_rwsem); 253 254 return rc; 255 } 256 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL"); 257 258 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 259 { 260 struct cxl_dev_state *cxlds = cxlmd->cxlds; 261 262 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 263 return 0; 264 265 if (!resource_size(&cxlds->dpa_res)) { 266 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 267 return -EINVAL; 268 } 269 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 270 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 271 dpa, &cxlds->dpa_res); 272 return -EINVAL; 273 } 274 if (!IS_ALIGNED(dpa, 64)) { 275 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 276 return -EINVAL; 277 } 278 279 return 0; 280 } 281 282 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 283 { 284 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 285 struct cxl_mbox_inject_poison inject; 286 struct cxl_poison_record record; 287 struct cxl_mbox_cmd mbox_cmd; 288 struct cxl_region *cxlr; 289 int rc; 290 291 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 292 return 0; 293 294 rc = down_read_interruptible(&cxl_region_rwsem); 295 if (rc) 296 return rc; 297 298 rc = down_read_interruptible(&cxl_dpa_rwsem); 299 if (rc) { 300 up_read(&cxl_region_rwsem); 301 return rc; 302 } 303 304 rc = cxl_validate_poison_dpa(cxlmd, dpa); 305 if (rc) 306 goto out; 307 308 inject.address = cpu_to_le64(dpa); 309 mbox_cmd = (struct cxl_mbox_cmd) { 310 .opcode = CXL_MBOX_OP_INJECT_POISON, 311 .size_in = sizeof(inject), 312 .payload_in = &inject, 313 }; 314 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 315 if (rc) 316 goto out; 317 318 cxlr = cxl_dpa_to_region(cxlmd, dpa); 319 if (cxlr) 320 dev_warn_once(cxl_mbox->host, 321 "poison inject dpa:%#llx region: %s\n", dpa, 322 dev_name(&cxlr->dev)); 323 324 record = (struct cxl_poison_record) { 325 .address = cpu_to_le64(dpa), 326 .length = cpu_to_le32(1), 327 }; 328 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 329 out: 330 up_read(&cxl_dpa_rwsem); 331 up_read(&cxl_region_rwsem); 332 333 return rc; 334 } 335 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL"); 336 337 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 338 { 339 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 340 struct cxl_mbox_clear_poison clear; 341 struct cxl_poison_record record; 342 struct cxl_mbox_cmd mbox_cmd; 343 struct cxl_region *cxlr; 344 int rc; 345 346 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 347 return 0; 348 349 rc = down_read_interruptible(&cxl_region_rwsem); 350 if (rc) 351 return rc; 352 353 rc = down_read_interruptible(&cxl_dpa_rwsem); 354 if (rc) { 355 up_read(&cxl_region_rwsem); 356 return rc; 357 } 358 359 rc = cxl_validate_poison_dpa(cxlmd, dpa); 360 if (rc) 361 goto out; 362 363 /* 364 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 365 * is defined to accept 64 bytes of write-data, along with the 366 * address to clear. This driver uses zeroes as write-data. 367 */ 368 clear = (struct cxl_mbox_clear_poison) { 369 .address = cpu_to_le64(dpa) 370 }; 371 372 mbox_cmd = (struct cxl_mbox_cmd) { 373 .opcode = CXL_MBOX_OP_CLEAR_POISON, 374 .size_in = sizeof(clear), 375 .payload_in = &clear, 376 }; 377 378 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 379 if (rc) 380 goto out; 381 382 cxlr = cxl_dpa_to_region(cxlmd, dpa); 383 if (cxlr) 384 dev_warn_once(cxl_mbox->host, 385 "poison clear dpa:%#llx region: %s\n", dpa, 386 dev_name(&cxlr->dev)); 387 388 record = (struct cxl_poison_record) { 389 .address = cpu_to_le64(dpa), 390 .length = cpu_to_le32(1), 391 }; 392 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 393 out: 394 up_read(&cxl_dpa_rwsem); 395 up_read(&cxl_region_rwsem); 396 397 return rc; 398 } 399 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL"); 400 401 static struct attribute *cxl_memdev_attributes[] = { 402 &dev_attr_serial.attr, 403 &dev_attr_firmware_version.attr, 404 &dev_attr_payload_max.attr, 405 &dev_attr_label_storage_size.attr, 406 &dev_attr_numa_node.attr, 407 NULL, 408 }; 409 410 static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds) 411 { 412 for (int i = 0; i < cxlds->nr_partitions; i++) 413 if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) 414 return &cxlds->part[i].perf; 415 return NULL; 416 } 417 418 static ssize_t pmem_qos_class_show(struct device *dev, 419 struct device_attribute *attr, char *buf) 420 { 421 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 422 struct cxl_dev_state *cxlds = cxlmd->cxlds; 423 424 return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class); 425 } 426 427 static struct device_attribute dev_attr_pmem_qos_class = 428 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); 429 430 static struct attribute *cxl_memdev_pmem_attributes[] = { 431 &dev_attr_pmem_size.attr, 432 &dev_attr_pmem_qos_class.attr, 433 NULL, 434 }; 435 436 static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds) 437 { 438 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 439 return NULL; 440 return &cxlds->part[0].perf; 441 } 442 443 static ssize_t ram_qos_class_show(struct device *dev, 444 struct device_attribute *attr, char *buf) 445 { 446 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 447 struct cxl_dev_state *cxlds = cxlmd->cxlds; 448 449 return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class); 450 } 451 452 static struct device_attribute dev_attr_ram_qos_class = 453 __ATTR(qos_class, 0444, ram_qos_class_show, NULL); 454 455 static struct attribute *cxl_memdev_ram_attributes[] = { 456 &dev_attr_ram_size.attr, 457 &dev_attr_ram_qos_class.attr, 458 NULL, 459 }; 460 461 static struct attribute *cxl_memdev_security_attributes[] = { 462 &dev_attr_security_state.attr, 463 &dev_attr_security_sanitize.attr, 464 &dev_attr_security_erase.attr, 465 NULL, 466 }; 467 468 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 469 int n) 470 { 471 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 472 return 0; 473 return a->mode; 474 } 475 476 static struct attribute_group cxl_memdev_attribute_group = { 477 .attrs = cxl_memdev_attributes, 478 .is_visible = cxl_memdev_visible, 479 }; 480 481 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) 482 { 483 struct device *dev = kobj_to_dev(kobj); 484 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 485 struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds); 486 487 if (a == &dev_attr_ram_qos_class.attr && 488 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 489 return 0; 490 491 return a->mode; 492 } 493 494 static struct attribute_group cxl_memdev_ram_attribute_group = { 495 .name = "ram", 496 .attrs = cxl_memdev_ram_attributes, 497 .is_visible = cxl_ram_visible, 498 }; 499 500 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) 501 { 502 struct device *dev = kobj_to_dev(kobj); 503 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 504 struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds); 505 506 if (a == &dev_attr_pmem_qos_class.attr && 507 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 508 return 0; 509 510 return a->mode; 511 } 512 513 static struct attribute_group cxl_memdev_pmem_attribute_group = { 514 .name = "pmem", 515 .attrs = cxl_memdev_pmem_attributes, 516 .is_visible = cxl_pmem_visible, 517 }; 518 519 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 520 struct attribute *a, int n) 521 { 522 struct device *dev = kobj_to_dev(kobj); 523 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 524 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 525 526 if (a == &dev_attr_security_sanitize.attr && 527 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 528 return 0; 529 530 if (a == &dev_attr_security_erase.attr && 531 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 532 return 0; 533 534 return a->mode; 535 } 536 537 static struct attribute_group cxl_memdev_security_attribute_group = { 538 .name = "security", 539 .attrs = cxl_memdev_security_attributes, 540 .is_visible = cxl_memdev_security_visible, 541 }; 542 543 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 544 &cxl_memdev_attribute_group, 545 &cxl_memdev_ram_attribute_group, 546 &cxl_memdev_pmem_attribute_group, 547 &cxl_memdev_security_attribute_group, 548 NULL, 549 }; 550 551 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) 552 { 553 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); 554 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); 555 } 556 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL"); 557 558 static const struct device_type cxl_memdev_type = { 559 .name = "cxl_memdev", 560 .release = cxl_memdev_release, 561 .devnode = cxl_memdev_devnode, 562 .groups = cxl_memdev_attribute_groups, 563 }; 564 565 bool is_cxl_memdev(const struct device *dev) 566 { 567 return dev->type == &cxl_memdev_type; 568 } 569 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); 570 571 /** 572 * set_exclusive_cxl_commands() - atomically disable user cxl commands 573 * @mds: The device state to operate on 574 * @cmds: bitmap of commands to mark exclusive 575 * 576 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 577 * invocations of the ioctl path and then disable future execution of 578 * commands with the command ids set in @cmds. 579 */ 580 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 581 unsigned long *cmds) 582 { 583 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 584 585 guard(rwsem_write)(&cxl_memdev_rwsem); 586 bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 587 cmds, CXL_MEM_COMMAND_ID_MAX); 588 } 589 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL"); 590 591 /** 592 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 593 * @mds: The device state to modify 594 * @cmds: bitmap of commands to mark available for userspace 595 */ 596 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 597 unsigned long *cmds) 598 { 599 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 600 601 guard(rwsem_write)(&cxl_memdev_rwsem); 602 bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 603 cmds, CXL_MEM_COMMAND_ID_MAX); 604 } 605 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL"); 606 607 static void cxl_memdev_shutdown(struct device *dev) 608 { 609 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 610 611 guard(rwsem_write)(&cxl_memdev_rwsem); 612 cxlmd->cxlds = NULL; 613 } 614 615 static void cxl_memdev_unregister(void *_cxlmd) 616 { 617 struct cxl_memdev *cxlmd = _cxlmd; 618 struct device *dev = &cxlmd->dev; 619 620 cdev_device_del(&cxlmd->cdev, dev); 621 cxl_memdev_shutdown(dev); 622 put_device(dev); 623 } 624 625 static void detach_memdev(struct work_struct *work) 626 { 627 struct cxl_memdev *cxlmd; 628 629 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 630 device_release_driver(&cxlmd->dev); 631 put_device(&cxlmd->dev); 632 } 633 634 static struct lock_class_key cxl_memdev_key; 635 636 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 637 const struct file_operations *fops) 638 { 639 struct cxl_memdev *cxlmd; 640 struct device *dev; 641 struct cdev *cdev; 642 int rc; 643 644 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 645 if (!cxlmd) 646 return ERR_PTR(-ENOMEM); 647 648 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 649 if (rc < 0) 650 goto err; 651 cxlmd->id = rc; 652 cxlmd->depth = -1; 653 654 dev = &cxlmd->dev; 655 device_initialize(dev); 656 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 657 dev->parent = cxlds->dev; 658 dev->bus = &cxl_bus_type; 659 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 660 dev->type = &cxl_memdev_type; 661 device_set_pm_not_required(dev); 662 INIT_WORK(&cxlmd->detach_work, detach_memdev); 663 664 cdev = &cxlmd->cdev; 665 cdev_init(cdev, fops); 666 return cxlmd; 667 668 err: 669 kfree(cxlmd); 670 return ERR_PTR(rc); 671 } 672 673 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 674 unsigned long arg) 675 { 676 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 677 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 678 679 switch (cmd) { 680 case CXL_MEM_QUERY_COMMANDS: 681 return cxl_query_cmd(cxl_mbox, (void __user *)arg); 682 case CXL_MEM_SEND_COMMAND: 683 return cxl_send_cmd(cxl_mbox, (void __user *)arg); 684 default: 685 return -ENOTTY; 686 } 687 } 688 689 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 690 unsigned long arg) 691 { 692 struct cxl_memdev *cxlmd = file->private_data; 693 struct cxl_dev_state *cxlds; 694 695 guard(rwsem_read)(&cxl_memdev_rwsem); 696 cxlds = cxlmd->cxlds; 697 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 698 return __cxl_memdev_ioctl(cxlmd, cmd, arg); 699 700 return -ENXIO; 701 } 702 703 static int cxl_memdev_open(struct inode *inode, struct file *file) 704 { 705 struct cxl_memdev *cxlmd = 706 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 707 708 get_device(&cxlmd->dev); 709 file->private_data = cxlmd; 710 711 return 0; 712 } 713 714 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 715 { 716 struct cxl_memdev *cxlmd = 717 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 718 719 put_device(&cxlmd->dev); 720 721 return 0; 722 } 723 724 /** 725 * cxl_mem_get_fw_info - Get Firmware info 726 * @mds: The device data for the operation 727 * 728 * Retrieve firmware info for the device specified. 729 * 730 * Return: 0 if no error: or the result of the mailbox command. 731 * 732 * See CXL-3.0 8.2.9.3.1 Get FW Info 733 */ 734 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 735 { 736 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 737 struct cxl_mbox_get_fw_info info; 738 struct cxl_mbox_cmd mbox_cmd; 739 int rc; 740 741 mbox_cmd = (struct cxl_mbox_cmd) { 742 .opcode = CXL_MBOX_OP_GET_FW_INFO, 743 .size_out = sizeof(info), 744 .payload_out = &info, 745 }; 746 747 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 748 if (rc < 0) 749 return rc; 750 751 mds->fw.num_slots = info.num_slots; 752 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 753 info.slot_info); 754 755 return 0; 756 } 757 758 /** 759 * cxl_mem_activate_fw - Activate Firmware 760 * @mds: The device data for the operation 761 * @slot: slot number to activate 762 * 763 * Activate firmware in a given slot for the device specified. 764 * 765 * Return: 0 if no error: or the result of the mailbox command. 766 * 767 * See CXL-3.0 8.2.9.3.3 Activate FW 768 */ 769 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 770 { 771 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 772 struct cxl_mbox_activate_fw activate; 773 struct cxl_mbox_cmd mbox_cmd; 774 775 if (slot == 0 || slot > mds->fw.num_slots) 776 return -EINVAL; 777 778 mbox_cmd = (struct cxl_mbox_cmd) { 779 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 780 .size_in = sizeof(activate), 781 .payload_in = &activate, 782 }; 783 784 /* Only offline activation supported for now */ 785 activate.action = CXL_FW_ACTIVATE_OFFLINE; 786 activate.slot = slot; 787 788 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 789 } 790 791 /** 792 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 793 * @mds: The device data for the operation 794 * 795 * Abort an in-progress firmware transfer for the device specified. 796 * 797 * Return: 0 if no error: or the result of the mailbox command. 798 * 799 * See CXL-3.0 8.2.9.3.2 Transfer FW 800 */ 801 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 802 { 803 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 804 struct cxl_mbox_transfer_fw *transfer; 805 struct cxl_mbox_cmd mbox_cmd; 806 int rc; 807 808 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 809 if (!transfer) 810 return -ENOMEM; 811 812 /* Set a 1s poll interval and a total wait time of 30s */ 813 mbox_cmd = (struct cxl_mbox_cmd) { 814 .opcode = CXL_MBOX_OP_TRANSFER_FW, 815 .size_in = sizeof(*transfer), 816 .payload_in = transfer, 817 .poll_interval_ms = 1000, 818 .poll_count = 30, 819 }; 820 821 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 822 823 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 824 kfree(transfer); 825 return rc; 826 } 827 828 static void cxl_fw_cleanup(struct fw_upload *fwl) 829 { 830 struct cxl_memdev_state *mds = fwl->dd_handle; 831 832 mds->fw.next_slot = 0; 833 } 834 835 static int cxl_fw_do_cancel(struct fw_upload *fwl) 836 { 837 struct cxl_memdev_state *mds = fwl->dd_handle; 838 struct cxl_dev_state *cxlds = &mds->cxlds; 839 struct cxl_memdev *cxlmd = cxlds->cxlmd; 840 int rc; 841 842 rc = cxl_mem_abort_fw_xfer(mds); 843 if (rc < 0) 844 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 845 846 return FW_UPLOAD_ERR_CANCELED; 847 } 848 849 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 850 u32 size) 851 { 852 struct cxl_memdev_state *mds = fwl->dd_handle; 853 struct cxl_mbox_transfer_fw *transfer; 854 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 855 856 if (!size) 857 return FW_UPLOAD_ERR_INVALID_SIZE; 858 859 mds->fw.oneshot = struct_size(transfer, data, size) < 860 cxl_mbox->payload_size; 861 862 if (cxl_mem_get_fw_info(mds)) 863 return FW_UPLOAD_ERR_HW_ERROR; 864 865 /* 866 * So far no state has been changed, hence no other cleanup is 867 * necessary. Simply return the cancelled status. 868 */ 869 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 870 return FW_UPLOAD_ERR_CANCELED; 871 872 return FW_UPLOAD_ERR_NONE; 873 } 874 875 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 876 u32 offset, u32 size, u32 *written) 877 { 878 struct cxl_memdev_state *mds = fwl->dd_handle; 879 struct cxl_dev_state *cxlds = &mds->cxlds; 880 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 881 struct cxl_memdev *cxlmd = cxlds->cxlmd; 882 struct cxl_mbox_transfer_fw *transfer; 883 struct cxl_mbox_cmd mbox_cmd; 884 u32 cur_size, remaining; 885 size_t size_in; 886 int rc; 887 888 *written = 0; 889 890 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 891 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 892 dev_err(&cxlmd->dev, 893 "misaligned offset for FW transfer slice (%u)\n", 894 offset); 895 return FW_UPLOAD_ERR_RW_ERROR; 896 } 897 898 /* 899 * Pick transfer size based on mds->payload_size @size must bw 128-byte 900 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 901 * sizeof(*transfer) is 128. These constraints imply that @cur_size 902 * will always be 128b aligned. 903 */ 904 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer)); 905 906 remaining = size - cur_size; 907 size_in = struct_size(transfer, data, cur_size); 908 909 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 910 return cxl_fw_do_cancel(fwl); 911 912 /* 913 * Slot numbers are 1-indexed 914 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 915 * Check for rollover using modulo, and 1-index it by adding 1 916 */ 917 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 918 919 /* Do the transfer via mailbox cmd */ 920 transfer = kzalloc(size_in, GFP_KERNEL); 921 if (!transfer) 922 return FW_UPLOAD_ERR_RW_ERROR; 923 924 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 925 memcpy(transfer->data, data + offset, cur_size); 926 if (mds->fw.oneshot) { 927 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 928 transfer->slot = mds->fw.next_slot; 929 } else { 930 if (offset == 0) { 931 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 932 } else if (remaining == 0) { 933 transfer->action = CXL_FW_TRANSFER_ACTION_END; 934 transfer->slot = mds->fw.next_slot; 935 } else { 936 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 937 } 938 } 939 940 mbox_cmd = (struct cxl_mbox_cmd) { 941 .opcode = CXL_MBOX_OP_TRANSFER_FW, 942 .size_in = size_in, 943 .payload_in = transfer, 944 .poll_interval_ms = 1000, 945 .poll_count = 30, 946 }; 947 948 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 949 if (rc < 0) { 950 rc = FW_UPLOAD_ERR_RW_ERROR; 951 goto out_free; 952 } 953 954 *written = cur_size; 955 956 /* Activate FW if oneshot or if the last slice was written */ 957 if (mds->fw.oneshot || remaining == 0) { 958 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 959 mds->fw.next_slot); 960 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 961 if (rc < 0) { 962 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 963 rc); 964 rc = FW_UPLOAD_ERR_HW_ERROR; 965 goto out_free; 966 } 967 } 968 969 rc = FW_UPLOAD_ERR_NONE; 970 971 out_free: 972 kfree(transfer); 973 return rc; 974 } 975 976 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 977 { 978 struct cxl_memdev_state *mds = fwl->dd_handle; 979 980 /* 981 * cxl_internal_send_cmd() handles background operations synchronously. 982 * No need to wait for completions here - any errors would've been 983 * reported and handled during the ->write() call(s). 984 * Just check if a cancel request was received, and return success. 985 */ 986 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 987 return cxl_fw_do_cancel(fwl); 988 989 return FW_UPLOAD_ERR_NONE; 990 } 991 992 static void cxl_fw_cancel(struct fw_upload *fwl) 993 { 994 struct cxl_memdev_state *mds = fwl->dd_handle; 995 996 set_bit(CXL_FW_CANCEL, mds->fw.state); 997 } 998 999 static const struct fw_upload_ops cxl_memdev_fw_ops = { 1000 .prepare = cxl_fw_prepare, 1001 .write = cxl_fw_write, 1002 .poll_complete = cxl_fw_poll_complete, 1003 .cancel = cxl_fw_cancel, 1004 .cleanup = cxl_fw_cleanup, 1005 }; 1006 1007 static void cxl_remove_fw_upload(void *fwl) 1008 { 1009 firmware_upload_unregister(fwl); 1010 } 1011 1012 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 1013 { 1014 struct cxl_dev_state *cxlds = &mds->cxlds; 1015 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 1016 struct device *dev = &cxlds->cxlmd->dev; 1017 struct fw_upload *fwl; 1018 1019 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds)) 1020 return 0; 1021 1022 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 1023 &cxl_memdev_fw_ops, mds); 1024 if (IS_ERR(fwl)) 1025 return PTR_ERR(fwl); 1026 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 1027 } 1028 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL"); 1029 1030 static const struct file_operations cxl_memdev_fops = { 1031 .owner = THIS_MODULE, 1032 .unlocked_ioctl = cxl_memdev_ioctl, 1033 .open = cxl_memdev_open, 1034 .release = cxl_memdev_release_file, 1035 .compat_ioctl = compat_ptr_ioctl, 1036 .llseek = noop_llseek, 1037 }; 1038 1039 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 1040 struct cxl_dev_state *cxlds) 1041 { 1042 struct cxl_memdev *cxlmd; 1043 struct device *dev; 1044 struct cdev *cdev; 1045 int rc; 1046 1047 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1048 if (IS_ERR(cxlmd)) 1049 return cxlmd; 1050 1051 dev = &cxlmd->dev; 1052 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1053 if (rc) 1054 goto err; 1055 1056 /* 1057 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1058 * needed as this is ordered with cdev_add() publishing the device. 1059 */ 1060 cxlmd->cxlds = cxlds; 1061 cxlds->cxlmd = cxlmd; 1062 1063 cdev = &cxlmd->cdev; 1064 rc = cdev_device_add(cdev, dev); 1065 if (rc) 1066 goto err; 1067 1068 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1069 if (rc) 1070 return ERR_PTR(rc); 1071 return cxlmd; 1072 1073 err: 1074 /* 1075 * The cdev was briefly live, shutdown any ioctl operations that 1076 * saw that state. 1077 */ 1078 cxl_memdev_shutdown(dev); 1079 put_device(dev); 1080 return ERR_PTR(rc); 1081 } 1082 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL"); 1083 1084 static void sanitize_teardown_notifier(void *data) 1085 { 1086 struct cxl_memdev_state *mds = data; 1087 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 1088 struct kernfs_node *state; 1089 1090 /* 1091 * Prevent new irq triggered invocations of the workqueue and 1092 * flush inflight invocations. 1093 */ 1094 mutex_lock(&cxl_mbox->mbox_mutex); 1095 state = mds->security.sanitize_node; 1096 mds->security.sanitize_node = NULL; 1097 mutex_unlock(&cxl_mbox->mbox_mutex); 1098 1099 cancel_delayed_work_sync(&mds->security.poll_dwork); 1100 sysfs_put(state); 1101 } 1102 1103 int devm_cxl_sanitize_setup_notifier(struct device *host, 1104 struct cxl_memdev *cxlmd) 1105 { 1106 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1107 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1108 struct kernfs_node *sec; 1109 1110 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1111 return 0; 1112 1113 /* 1114 * Note, the expectation is that @cxlmd would have failed to be 1115 * created if these sysfs_get_dirent calls fail. 1116 */ 1117 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1118 if (!sec) 1119 return -ENOENT; 1120 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1121 sysfs_put(sec); 1122 if (!mds->security.sanitize_node) 1123 return -ENOENT; 1124 1125 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1126 } 1127 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL"); 1128 1129 __init int cxl_memdev_init(void) 1130 { 1131 dev_t devt; 1132 int rc; 1133 1134 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1135 if (rc) 1136 return rc; 1137 1138 cxl_mem_major = MAJOR(devt); 1139 1140 return 0; 1141 } 1142 1143 void cxl_memdev_exit(void) 1144 { 1145 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1146 } 1147