1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 kfree(cxlmd); 31 } 32 33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 34 kgid_t *gid) 35 { 36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 37 } 38 39 static ssize_t firmware_version_show(struct device *dev, 40 struct device_attribute *attr, char *buf) 41 { 42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 43 struct cxl_dev_state *cxlds = cxlmd->cxlds; 44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 45 46 if (!mds) 47 return sysfs_emit(buf, "\n"); 48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 49 } 50 static DEVICE_ATTR_RO(firmware_version); 51 52 static ssize_t payload_max_show(struct device *dev, 53 struct device_attribute *attr, char *buf) 54 { 55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 56 struct cxl_dev_state *cxlds = cxlmd->cxlds; 57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 58 59 if (!mds) 60 return sysfs_emit(buf, "\n"); 61 return sysfs_emit(buf, "%zu\n", mds->payload_size); 62 } 63 static DEVICE_ATTR_RO(payload_max); 64 65 static ssize_t label_storage_size_show(struct device *dev, 66 struct device_attribute *attr, char *buf) 67 { 68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 69 struct cxl_dev_state *cxlds = cxlmd->cxlds; 70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 71 72 if (!mds) 73 return sysfs_emit(buf, "\n"); 74 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 75 } 76 static DEVICE_ATTR_RO(label_storage_size); 77 78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 79 char *buf) 80 { 81 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 82 struct cxl_dev_state *cxlds = cxlmd->cxlds; 83 unsigned long long len = resource_size(&cxlds->ram_res); 84 85 return sysfs_emit(buf, "%#llx\n", len); 86 } 87 88 static struct device_attribute dev_attr_ram_size = 89 __ATTR(size, 0444, ram_size_show, NULL); 90 91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 92 char *buf) 93 { 94 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 95 struct cxl_dev_state *cxlds = cxlmd->cxlds; 96 unsigned long long len = resource_size(&cxlds->pmem_res); 97 98 return sysfs_emit(buf, "%#llx\n", len); 99 } 100 101 static struct device_attribute dev_attr_pmem_size = 102 __ATTR(size, 0444, pmem_size_show, NULL); 103 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 105 char *buf) 106 { 107 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 108 struct cxl_dev_state *cxlds = cxlmd->cxlds; 109 110 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 111 } 112 static DEVICE_ATTR_RO(serial); 113 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 115 char *buf) 116 { 117 return sysfs_emit(buf, "%d\n", dev_to_node(dev)); 118 } 119 static DEVICE_ATTR_RO(numa_node); 120 121 static ssize_t security_state_show(struct device *dev, 122 struct device_attribute *attr, 123 char *buf) 124 { 125 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 126 struct cxl_dev_state *cxlds = cxlmd->cxlds; 127 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 128 unsigned long state = mds->security.state; 129 int rc = 0; 130 131 /* sync with latest submission state */ 132 mutex_lock(&mds->mbox_mutex); 133 if (mds->security.sanitize_active) 134 rc = sysfs_emit(buf, "sanitize\n"); 135 mutex_unlock(&mds->mbox_mutex); 136 if (rc) 137 return rc; 138 139 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 140 return sysfs_emit(buf, "disabled\n"); 141 if (state & CXL_PMEM_SEC_STATE_FROZEN || 142 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 143 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 144 return sysfs_emit(buf, "frozen\n"); 145 if (state & CXL_PMEM_SEC_STATE_LOCKED) 146 return sysfs_emit(buf, "locked\n"); 147 else 148 return sysfs_emit(buf, "unlocked\n"); 149 } 150 static struct device_attribute dev_attr_security_state = 151 __ATTR(state, 0444, security_state_show, NULL); 152 153 static ssize_t security_sanitize_store(struct device *dev, 154 struct device_attribute *attr, 155 const char *buf, size_t len) 156 { 157 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 158 bool sanitize; 159 ssize_t rc; 160 161 if (kstrtobool(buf, &sanitize) || !sanitize) 162 return -EINVAL; 163 164 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 165 if (rc) 166 return rc; 167 168 return len; 169 } 170 static struct device_attribute dev_attr_security_sanitize = 171 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 172 173 static ssize_t security_erase_store(struct device *dev, 174 struct device_attribute *attr, 175 const char *buf, size_t len) 176 { 177 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 178 ssize_t rc; 179 bool erase; 180 181 if (kstrtobool(buf, &erase) || !erase) 182 return -EINVAL; 183 184 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 185 if (rc) 186 return rc; 187 188 return len; 189 } 190 static struct device_attribute dev_attr_security_erase = 191 __ATTR(erase, 0200, NULL, security_erase_store); 192 193 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 194 { 195 struct cxl_dev_state *cxlds = cxlmd->cxlds; 196 u64 offset, length; 197 int rc = 0; 198 199 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 200 if (resource_size(&cxlds->pmem_res)) { 201 offset = cxlds->pmem_res.start; 202 length = resource_size(&cxlds->pmem_res); 203 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 204 if (rc) 205 return rc; 206 } 207 if (resource_size(&cxlds->ram_res)) { 208 offset = cxlds->ram_res.start; 209 length = resource_size(&cxlds->ram_res); 210 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 211 /* 212 * Invalid Physical Address is not an error for 213 * volatile addresses. Device support is optional. 214 */ 215 if (rc == -EFAULT) 216 rc = 0; 217 } 218 return rc; 219 } 220 221 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 222 { 223 struct cxl_port *port; 224 int rc; 225 226 port = cxlmd->endpoint; 227 if (!port || !is_cxl_endpoint(port)) 228 return -EINVAL; 229 230 rc = down_read_interruptible(&cxl_region_rwsem); 231 if (rc) 232 return rc; 233 234 rc = down_read_interruptible(&cxl_dpa_rwsem); 235 if (rc) { 236 up_read(&cxl_region_rwsem); 237 return rc; 238 } 239 240 if (cxl_num_decoders_committed(port) == 0) { 241 /* No regions mapped to this memdev */ 242 rc = cxl_get_poison_by_memdev(cxlmd); 243 } else { 244 /* Regions mapped, collect poison by endpoint */ 245 rc = cxl_get_poison_by_endpoint(port); 246 } 247 up_read(&cxl_dpa_rwsem); 248 up_read(&cxl_region_rwsem); 249 250 return rc; 251 } 252 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL); 253 254 struct cxl_dpa_to_region_context { 255 struct cxl_region *cxlr; 256 u64 dpa; 257 }; 258 259 static int __cxl_dpa_to_region(struct device *dev, void *arg) 260 { 261 struct cxl_dpa_to_region_context *ctx = arg; 262 struct cxl_endpoint_decoder *cxled; 263 u64 dpa = ctx->dpa; 264 265 if (!is_endpoint_decoder(dev)) 266 return 0; 267 268 cxled = to_cxl_endpoint_decoder(dev); 269 if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) 270 return 0; 271 272 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start) 273 return 0; 274 275 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa, 276 dev_name(&cxled->cxld.region->dev)); 277 278 ctx->cxlr = cxled->cxld.region; 279 280 return 1; 281 } 282 283 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa) 284 { 285 struct cxl_dpa_to_region_context ctx; 286 struct cxl_port *port; 287 288 ctx = (struct cxl_dpa_to_region_context) { 289 .dpa = dpa, 290 }; 291 port = cxlmd->endpoint; 292 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port)) 293 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); 294 295 return ctx.cxlr; 296 } 297 298 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 299 { 300 struct cxl_dev_state *cxlds = cxlmd->cxlds; 301 302 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 303 return 0; 304 305 if (!resource_size(&cxlds->dpa_res)) { 306 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 307 return -EINVAL; 308 } 309 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 310 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 311 dpa, &cxlds->dpa_res); 312 return -EINVAL; 313 } 314 if (!IS_ALIGNED(dpa, 64)) { 315 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 316 return -EINVAL; 317 } 318 319 return 0; 320 } 321 322 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 323 { 324 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 325 struct cxl_mbox_inject_poison inject; 326 struct cxl_poison_record record; 327 struct cxl_mbox_cmd mbox_cmd; 328 struct cxl_region *cxlr; 329 int rc; 330 331 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 332 return 0; 333 334 rc = down_read_interruptible(&cxl_region_rwsem); 335 if (rc) 336 return rc; 337 338 rc = down_read_interruptible(&cxl_dpa_rwsem); 339 if (rc) { 340 up_read(&cxl_region_rwsem); 341 return rc; 342 } 343 344 rc = cxl_validate_poison_dpa(cxlmd, dpa); 345 if (rc) 346 goto out; 347 348 inject.address = cpu_to_le64(dpa); 349 mbox_cmd = (struct cxl_mbox_cmd) { 350 .opcode = CXL_MBOX_OP_INJECT_POISON, 351 .size_in = sizeof(inject), 352 .payload_in = &inject, 353 }; 354 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 355 if (rc) 356 goto out; 357 358 cxlr = cxl_dpa_to_region(cxlmd, dpa); 359 if (cxlr) 360 dev_warn_once(mds->cxlds.dev, 361 "poison inject dpa:%#llx region: %s\n", dpa, 362 dev_name(&cxlr->dev)); 363 364 record = (struct cxl_poison_record) { 365 .address = cpu_to_le64(dpa), 366 .length = cpu_to_le32(1), 367 }; 368 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 369 out: 370 up_read(&cxl_dpa_rwsem); 371 up_read(&cxl_region_rwsem); 372 373 return rc; 374 } 375 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL); 376 377 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 378 { 379 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 380 struct cxl_mbox_clear_poison clear; 381 struct cxl_poison_record record; 382 struct cxl_mbox_cmd mbox_cmd; 383 struct cxl_region *cxlr; 384 int rc; 385 386 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 387 return 0; 388 389 rc = down_read_interruptible(&cxl_region_rwsem); 390 if (rc) 391 return rc; 392 393 rc = down_read_interruptible(&cxl_dpa_rwsem); 394 if (rc) { 395 up_read(&cxl_region_rwsem); 396 return rc; 397 } 398 399 rc = cxl_validate_poison_dpa(cxlmd, dpa); 400 if (rc) 401 goto out; 402 403 /* 404 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 405 * is defined to accept 64 bytes of write-data, along with the 406 * address to clear. This driver uses zeroes as write-data. 407 */ 408 clear = (struct cxl_mbox_clear_poison) { 409 .address = cpu_to_le64(dpa) 410 }; 411 412 mbox_cmd = (struct cxl_mbox_cmd) { 413 .opcode = CXL_MBOX_OP_CLEAR_POISON, 414 .size_in = sizeof(clear), 415 .payload_in = &clear, 416 }; 417 418 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 419 if (rc) 420 goto out; 421 422 cxlr = cxl_dpa_to_region(cxlmd, dpa); 423 if (cxlr) 424 dev_warn_once(mds->cxlds.dev, 425 "poison clear dpa:%#llx region: %s\n", dpa, 426 dev_name(&cxlr->dev)); 427 428 record = (struct cxl_poison_record) { 429 .address = cpu_to_le64(dpa), 430 .length = cpu_to_le32(1), 431 }; 432 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 433 out: 434 up_read(&cxl_dpa_rwsem); 435 up_read(&cxl_region_rwsem); 436 437 return rc; 438 } 439 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL); 440 441 static struct attribute *cxl_memdev_attributes[] = { 442 &dev_attr_serial.attr, 443 &dev_attr_firmware_version.attr, 444 &dev_attr_payload_max.attr, 445 &dev_attr_label_storage_size.attr, 446 &dev_attr_numa_node.attr, 447 NULL, 448 }; 449 450 static ssize_t pmem_qos_class_show(struct device *dev, 451 struct device_attribute *attr, char *buf) 452 { 453 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 454 struct cxl_dev_state *cxlds = cxlmd->cxlds; 455 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 456 457 return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class); 458 } 459 460 static struct device_attribute dev_attr_pmem_qos_class = 461 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); 462 463 static struct attribute *cxl_memdev_pmem_attributes[] = { 464 &dev_attr_pmem_size.attr, 465 &dev_attr_pmem_qos_class.attr, 466 NULL, 467 }; 468 469 static ssize_t ram_qos_class_show(struct device *dev, 470 struct device_attribute *attr, char *buf) 471 { 472 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 473 struct cxl_dev_state *cxlds = cxlmd->cxlds; 474 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 475 476 return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class); 477 } 478 479 static struct device_attribute dev_attr_ram_qos_class = 480 __ATTR(qos_class, 0444, ram_qos_class_show, NULL); 481 482 static struct attribute *cxl_memdev_ram_attributes[] = { 483 &dev_attr_ram_size.attr, 484 &dev_attr_ram_qos_class.attr, 485 NULL, 486 }; 487 488 static struct attribute *cxl_memdev_security_attributes[] = { 489 &dev_attr_security_state.attr, 490 &dev_attr_security_sanitize.attr, 491 &dev_attr_security_erase.attr, 492 NULL, 493 }; 494 495 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 496 int n) 497 { 498 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 499 return 0; 500 return a->mode; 501 } 502 503 static struct attribute_group cxl_memdev_attribute_group = { 504 .attrs = cxl_memdev_attributes, 505 .is_visible = cxl_memdev_visible, 506 }; 507 508 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) 509 { 510 struct device *dev = kobj_to_dev(kobj); 511 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 512 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 513 514 if (a == &dev_attr_ram_qos_class.attr) 515 if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID) 516 return 0; 517 518 return a->mode; 519 } 520 521 static struct attribute_group cxl_memdev_ram_attribute_group = { 522 .name = "ram", 523 .attrs = cxl_memdev_ram_attributes, 524 .is_visible = cxl_ram_visible, 525 }; 526 527 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) 528 { 529 struct device *dev = kobj_to_dev(kobj); 530 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 531 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 532 533 if (a == &dev_attr_pmem_qos_class.attr) 534 if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID) 535 return 0; 536 537 return a->mode; 538 } 539 540 static struct attribute_group cxl_memdev_pmem_attribute_group = { 541 .name = "pmem", 542 .attrs = cxl_memdev_pmem_attributes, 543 .is_visible = cxl_pmem_visible, 544 }; 545 546 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 547 struct attribute *a, int n) 548 { 549 struct device *dev = kobj_to_dev(kobj); 550 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 551 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 552 553 if (a == &dev_attr_security_sanitize.attr && 554 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 555 return 0; 556 557 if (a == &dev_attr_security_erase.attr && 558 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 559 return 0; 560 561 return a->mode; 562 } 563 564 static struct attribute_group cxl_memdev_security_attribute_group = { 565 .name = "security", 566 .attrs = cxl_memdev_security_attributes, 567 .is_visible = cxl_memdev_security_visible, 568 }; 569 570 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 571 &cxl_memdev_attribute_group, 572 &cxl_memdev_ram_attribute_group, 573 &cxl_memdev_pmem_attribute_group, 574 &cxl_memdev_security_attribute_group, 575 NULL, 576 }; 577 578 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) 579 { 580 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); 581 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); 582 } 583 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, CXL); 584 585 static const struct device_type cxl_memdev_type = { 586 .name = "cxl_memdev", 587 .release = cxl_memdev_release, 588 .devnode = cxl_memdev_devnode, 589 .groups = cxl_memdev_attribute_groups, 590 }; 591 592 bool is_cxl_memdev(const struct device *dev) 593 { 594 return dev->type == &cxl_memdev_type; 595 } 596 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL); 597 598 /** 599 * set_exclusive_cxl_commands() - atomically disable user cxl commands 600 * @mds: The device state to operate on 601 * @cmds: bitmap of commands to mark exclusive 602 * 603 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 604 * invocations of the ioctl path and then disable future execution of 605 * commands with the command ids set in @cmds. 606 */ 607 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 608 unsigned long *cmds) 609 { 610 down_write(&cxl_memdev_rwsem); 611 bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 612 CXL_MEM_COMMAND_ID_MAX); 613 up_write(&cxl_memdev_rwsem); 614 } 615 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL); 616 617 /** 618 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 619 * @mds: The device state to modify 620 * @cmds: bitmap of commands to mark available for userspace 621 */ 622 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 623 unsigned long *cmds) 624 { 625 down_write(&cxl_memdev_rwsem); 626 bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 627 CXL_MEM_COMMAND_ID_MAX); 628 up_write(&cxl_memdev_rwsem); 629 } 630 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL); 631 632 static void cxl_memdev_shutdown(struct device *dev) 633 { 634 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 635 636 down_write(&cxl_memdev_rwsem); 637 cxlmd->cxlds = NULL; 638 up_write(&cxl_memdev_rwsem); 639 } 640 641 static void cxl_memdev_unregister(void *_cxlmd) 642 { 643 struct cxl_memdev *cxlmd = _cxlmd; 644 struct device *dev = &cxlmd->dev; 645 646 cdev_device_del(&cxlmd->cdev, dev); 647 cxl_memdev_shutdown(dev); 648 put_device(dev); 649 } 650 651 static void detach_memdev(struct work_struct *work) 652 { 653 struct cxl_memdev *cxlmd; 654 655 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 656 device_release_driver(&cxlmd->dev); 657 put_device(&cxlmd->dev); 658 } 659 660 static struct lock_class_key cxl_memdev_key; 661 662 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 663 const struct file_operations *fops) 664 { 665 struct cxl_memdev *cxlmd; 666 struct device *dev; 667 struct cdev *cdev; 668 int rc; 669 670 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 671 if (!cxlmd) 672 return ERR_PTR(-ENOMEM); 673 674 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 675 if (rc < 0) 676 goto err; 677 cxlmd->id = rc; 678 cxlmd->depth = -1; 679 680 dev = &cxlmd->dev; 681 device_initialize(dev); 682 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 683 dev->parent = cxlds->dev; 684 dev->bus = &cxl_bus_type; 685 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 686 dev->type = &cxl_memdev_type; 687 device_set_pm_not_required(dev); 688 INIT_WORK(&cxlmd->detach_work, detach_memdev); 689 690 cdev = &cxlmd->cdev; 691 cdev_init(cdev, fops); 692 return cxlmd; 693 694 err: 695 kfree(cxlmd); 696 return ERR_PTR(rc); 697 } 698 699 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 700 unsigned long arg) 701 { 702 switch (cmd) { 703 case CXL_MEM_QUERY_COMMANDS: 704 return cxl_query_cmd(cxlmd, (void __user *)arg); 705 case CXL_MEM_SEND_COMMAND: 706 return cxl_send_cmd(cxlmd, (void __user *)arg); 707 default: 708 return -ENOTTY; 709 } 710 } 711 712 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 713 unsigned long arg) 714 { 715 struct cxl_memdev *cxlmd = file->private_data; 716 struct cxl_dev_state *cxlds; 717 int rc = -ENXIO; 718 719 down_read(&cxl_memdev_rwsem); 720 cxlds = cxlmd->cxlds; 721 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 722 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 723 up_read(&cxl_memdev_rwsem); 724 725 return rc; 726 } 727 728 static int cxl_memdev_open(struct inode *inode, struct file *file) 729 { 730 struct cxl_memdev *cxlmd = 731 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 732 733 get_device(&cxlmd->dev); 734 file->private_data = cxlmd; 735 736 return 0; 737 } 738 739 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 740 { 741 struct cxl_memdev *cxlmd = 742 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 743 744 put_device(&cxlmd->dev); 745 746 return 0; 747 } 748 749 /** 750 * cxl_mem_get_fw_info - Get Firmware info 751 * @mds: The device data for the operation 752 * 753 * Retrieve firmware info for the device specified. 754 * 755 * Return: 0 if no error: or the result of the mailbox command. 756 * 757 * See CXL-3.0 8.2.9.3.1 Get FW Info 758 */ 759 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 760 { 761 struct cxl_mbox_get_fw_info info; 762 struct cxl_mbox_cmd mbox_cmd; 763 int rc; 764 765 mbox_cmd = (struct cxl_mbox_cmd) { 766 .opcode = CXL_MBOX_OP_GET_FW_INFO, 767 .size_out = sizeof(info), 768 .payload_out = &info, 769 }; 770 771 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 772 if (rc < 0) 773 return rc; 774 775 mds->fw.num_slots = info.num_slots; 776 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 777 info.slot_info); 778 779 return 0; 780 } 781 782 /** 783 * cxl_mem_activate_fw - Activate Firmware 784 * @mds: The device data for the operation 785 * @slot: slot number to activate 786 * 787 * Activate firmware in a given slot for the device specified. 788 * 789 * Return: 0 if no error: or the result of the mailbox command. 790 * 791 * See CXL-3.0 8.2.9.3.3 Activate FW 792 */ 793 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 794 { 795 struct cxl_mbox_activate_fw activate; 796 struct cxl_mbox_cmd mbox_cmd; 797 798 if (slot == 0 || slot > mds->fw.num_slots) 799 return -EINVAL; 800 801 mbox_cmd = (struct cxl_mbox_cmd) { 802 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 803 .size_in = sizeof(activate), 804 .payload_in = &activate, 805 }; 806 807 /* Only offline activation supported for now */ 808 activate.action = CXL_FW_ACTIVATE_OFFLINE; 809 activate.slot = slot; 810 811 return cxl_internal_send_cmd(mds, &mbox_cmd); 812 } 813 814 /** 815 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 816 * @mds: The device data for the operation 817 * 818 * Abort an in-progress firmware transfer for the device specified. 819 * 820 * Return: 0 if no error: or the result of the mailbox command. 821 * 822 * See CXL-3.0 8.2.9.3.2 Transfer FW 823 */ 824 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 825 { 826 struct cxl_mbox_transfer_fw *transfer; 827 struct cxl_mbox_cmd mbox_cmd; 828 int rc; 829 830 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 831 if (!transfer) 832 return -ENOMEM; 833 834 /* Set a 1s poll interval and a total wait time of 30s */ 835 mbox_cmd = (struct cxl_mbox_cmd) { 836 .opcode = CXL_MBOX_OP_TRANSFER_FW, 837 .size_in = sizeof(*transfer), 838 .payload_in = transfer, 839 .poll_interval_ms = 1000, 840 .poll_count = 30, 841 }; 842 843 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 844 845 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 846 kfree(transfer); 847 return rc; 848 } 849 850 static void cxl_fw_cleanup(struct fw_upload *fwl) 851 { 852 struct cxl_memdev_state *mds = fwl->dd_handle; 853 854 mds->fw.next_slot = 0; 855 } 856 857 static int cxl_fw_do_cancel(struct fw_upload *fwl) 858 { 859 struct cxl_memdev_state *mds = fwl->dd_handle; 860 struct cxl_dev_state *cxlds = &mds->cxlds; 861 struct cxl_memdev *cxlmd = cxlds->cxlmd; 862 int rc; 863 864 rc = cxl_mem_abort_fw_xfer(mds); 865 if (rc < 0) 866 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 867 868 return FW_UPLOAD_ERR_CANCELED; 869 } 870 871 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 872 u32 size) 873 { 874 struct cxl_memdev_state *mds = fwl->dd_handle; 875 struct cxl_mbox_transfer_fw *transfer; 876 877 if (!size) 878 return FW_UPLOAD_ERR_INVALID_SIZE; 879 880 mds->fw.oneshot = struct_size(transfer, data, size) < 881 mds->payload_size; 882 883 if (cxl_mem_get_fw_info(mds)) 884 return FW_UPLOAD_ERR_HW_ERROR; 885 886 /* 887 * So far no state has been changed, hence no other cleanup is 888 * necessary. Simply return the cancelled status. 889 */ 890 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 891 return FW_UPLOAD_ERR_CANCELED; 892 893 return FW_UPLOAD_ERR_NONE; 894 } 895 896 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 897 u32 offset, u32 size, u32 *written) 898 { 899 struct cxl_memdev_state *mds = fwl->dd_handle; 900 struct cxl_dev_state *cxlds = &mds->cxlds; 901 struct cxl_memdev *cxlmd = cxlds->cxlmd; 902 struct cxl_mbox_transfer_fw *transfer; 903 struct cxl_mbox_cmd mbox_cmd; 904 u32 cur_size, remaining; 905 size_t size_in; 906 int rc; 907 908 *written = 0; 909 910 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 911 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 912 dev_err(&cxlmd->dev, 913 "misaligned offset for FW transfer slice (%u)\n", 914 offset); 915 return FW_UPLOAD_ERR_RW_ERROR; 916 } 917 918 /* 919 * Pick transfer size based on mds->payload_size @size must bw 128-byte 920 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 921 * sizeof(*transfer) is 128. These constraints imply that @cur_size 922 * will always be 128b aligned. 923 */ 924 cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer)); 925 926 remaining = size - cur_size; 927 size_in = struct_size(transfer, data, cur_size); 928 929 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 930 return cxl_fw_do_cancel(fwl); 931 932 /* 933 * Slot numbers are 1-indexed 934 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 935 * Check for rollover using modulo, and 1-index it by adding 1 936 */ 937 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 938 939 /* Do the transfer via mailbox cmd */ 940 transfer = kzalloc(size_in, GFP_KERNEL); 941 if (!transfer) 942 return FW_UPLOAD_ERR_RW_ERROR; 943 944 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 945 memcpy(transfer->data, data + offset, cur_size); 946 if (mds->fw.oneshot) { 947 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 948 transfer->slot = mds->fw.next_slot; 949 } else { 950 if (offset == 0) { 951 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 952 } else if (remaining == 0) { 953 transfer->action = CXL_FW_TRANSFER_ACTION_END; 954 transfer->slot = mds->fw.next_slot; 955 } else { 956 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 957 } 958 } 959 960 mbox_cmd = (struct cxl_mbox_cmd) { 961 .opcode = CXL_MBOX_OP_TRANSFER_FW, 962 .size_in = size_in, 963 .payload_in = transfer, 964 .poll_interval_ms = 1000, 965 .poll_count = 30, 966 }; 967 968 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 969 if (rc < 0) { 970 rc = FW_UPLOAD_ERR_RW_ERROR; 971 goto out_free; 972 } 973 974 *written = cur_size; 975 976 /* Activate FW if oneshot or if the last slice was written */ 977 if (mds->fw.oneshot || remaining == 0) { 978 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 979 mds->fw.next_slot); 980 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 981 if (rc < 0) { 982 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 983 rc); 984 rc = FW_UPLOAD_ERR_HW_ERROR; 985 goto out_free; 986 } 987 } 988 989 rc = FW_UPLOAD_ERR_NONE; 990 991 out_free: 992 kfree(transfer); 993 return rc; 994 } 995 996 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 997 { 998 struct cxl_memdev_state *mds = fwl->dd_handle; 999 1000 /* 1001 * cxl_internal_send_cmd() handles background operations synchronously. 1002 * No need to wait for completions here - any errors would've been 1003 * reported and handled during the ->write() call(s). 1004 * Just check if a cancel request was received, and return success. 1005 */ 1006 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 1007 return cxl_fw_do_cancel(fwl); 1008 1009 return FW_UPLOAD_ERR_NONE; 1010 } 1011 1012 static void cxl_fw_cancel(struct fw_upload *fwl) 1013 { 1014 struct cxl_memdev_state *mds = fwl->dd_handle; 1015 1016 set_bit(CXL_FW_CANCEL, mds->fw.state); 1017 } 1018 1019 static const struct fw_upload_ops cxl_memdev_fw_ops = { 1020 .prepare = cxl_fw_prepare, 1021 .write = cxl_fw_write, 1022 .poll_complete = cxl_fw_poll_complete, 1023 .cancel = cxl_fw_cancel, 1024 .cleanup = cxl_fw_cleanup, 1025 }; 1026 1027 static void cxl_remove_fw_upload(void *fwl) 1028 { 1029 firmware_upload_unregister(fwl); 1030 } 1031 1032 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 1033 { 1034 struct cxl_dev_state *cxlds = &mds->cxlds; 1035 struct device *dev = &cxlds->cxlmd->dev; 1036 struct fw_upload *fwl; 1037 1038 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds)) 1039 return 0; 1040 1041 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 1042 &cxl_memdev_fw_ops, mds); 1043 if (IS_ERR(fwl)) 1044 return PTR_ERR(fwl); 1045 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 1046 } 1047 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, CXL); 1048 1049 static const struct file_operations cxl_memdev_fops = { 1050 .owner = THIS_MODULE, 1051 .unlocked_ioctl = cxl_memdev_ioctl, 1052 .open = cxl_memdev_open, 1053 .release = cxl_memdev_release_file, 1054 .compat_ioctl = compat_ptr_ioctl, 1055 .llseek = noop_llseek, 1056 }; 1057 1058 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 1059 struct cxl_dev_state *cxlds) 1060 { 1061 struct cxl_memdev *cxlmd; 1062 struct device *dev; 1063 struct cdev *cdev; 1064 int rc; 1065 1066 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1067 if (IS_ERR(cxlmd)) 1068 return cxlmd; 1069 1070 dev = &cxlmd->dev; 1071 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1072 if (rc) 1073 goto err; 1074 1075 /* 1076 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1077 * needed as this is ordered with cdev_add() publishing the device. 1078 */ 1079 cxlmd->cxlds = cxlds; 1080 cxlds->cxlmd = cxlmd; 1081 1082 cdev = &cxlmd->cdev; 1083 rc = cdev_device_add(cdev, dev); 1084 if (rc) 1085 goto err; 1086 1087 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1088 if (rc) 1089 return ERR_PTR(rc); 1090 return cxlmd; 1091 1092 err: 1093 /* 1094 * The cdev was briefly live, shutdown any ioctl operations that 1095 * saw that state. 1096 */ 1097 cxl_memdev_shutdown(dev); 1098 put_device(dev); 1099 return ERR_PTR(rc); 1100 } 1101 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL); 1102 1103 static void sanitize_teardown_notifier(void *data) 1104 { 1105 struct cxl_memdev_state *mds = data; 1106 struct kernfs_node *state; 1107 1108 /* 1109 * Prevent new irq triggered invocations of the workqueue and 1110 * flush inflight invocations. 1111 */ 1112 mutex_lock(&mds->mbox_mutex); 1113 state = mds->security.sanitize_node; 1114 mds->security.sanitize_node = NULL; 1115 mutex_unlock(&mds->mbox_mutex); 1116 1117 cancel_delayed_work_sync(&mds->security.poll_dwork); 1118 sysfs_put(state); 1119 } 1120 1121 int devm_cxl_sanitize_setup_notifier(struct device *host, 1122 struct cxl_memdev *cxlmd) 1123 { 1124 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1125 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1126 struct kernfs_node *sec; 1127 1128 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1129 return 0; 1130 1131 /* 1132 * Note, the expectation is that @cxlmd would have failed to be 1133 * created if these sysfs_get_dirent calls fail. 1134 */ 1135 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1136 if (!sec) 1137 return -ENOENT; 1138 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1139 sysfs_put(sec); 1140 if (!mds->security.sanitize_node) 1141 return -ENOENT; 1142 1143 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1144 } 1145 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, CXL); 1146 1147 __init int cxl_memdev_init(void) 1148 { 1149 dev_t devt; 1150 int rc; 1151 1152 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1153 if (rc) 1154 return rc; 1155 1156 cxl_mem_major = MAJOR(devt); 1157 1158 return 0; 1159 } 1160 1161 void cxl_memdev_exit(void) 1162 { 1163 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1164 } 1165