1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 devm_cxl_memdev_edac_release(cxlmd); 31 kfree(cxlmd); 32 } 33 34 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 35 kgid_t *gid) 36 { 37 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 38 } 39 40 static ssize_t firmware_version_show(struct device *dev, 41 struct device_attribute *attr, char *buf) 42 { 43 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 44 struct cxl_dev_state *cxlds = cxlmd->cxlds; 45 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 46 47 if (!mds) 48 return sysfs_emit(buf, "\n"); 49 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 50 } 51 static DEVICE_ATTR_RO(firmware_version); 52 53 static ssize_t payload_max_show(struct device *dev, 54 struct device_attribute *attr, char *buf) 55 { 56 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 57 struct cxl_dev_state *cxlds = cxlmd->cxlds; 58 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 59 60 if (!mds) 61 return sysfs_emit(buf, "\n"); 62 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size); 63 } 64 static DEVICE_ATTR_RO(payload_max); 65 66 static ssize_t label_storage_size_show(struct device *dev, 67 struct device_attribute *attr, char *buf) 68 { 69 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 70 struct cxl_dev_state *cxlds = cxlmd->cxlds; 71 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 72 73 if (!mds) 74 return sysfs_emit(buf, "\n"); 75 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 76 } 77 static DEVICE_ATTR_RO(label_storage_size); 78 79 static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds) 80 { 81 /* Static RAM is only expected at partition 0. */ 82 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 83 return 0; 84 return resource_size(&cxlds->part[0].res); 85 } 86 87 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 88 char *buf) 89 { 90 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 91 struct cxl_dev_state *cxlds = cxlmd->cxlds; 92 unsigned long long len = cxl_ram_size(cxlds); 93 94 return sysfs_emit(buf, "%#llx\n", len); 95 } 96 97 static struct device_attribute dev_attr_ram_size = 98 __ATTR(size, 0444, ram_size_show, NULL); 99 100 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 101 char *buf) 102 { 103 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 104 struct cxl_dev_state *cxlds = cxlmd->cxlds; 105 unsigned long long len = cxl_pmem_size(cxlds); 106 107 return sysfs_emit(buf, "%#llx\n", len); 108 } 109 110 static struct device_attribute dev_attr_pmem_size = 111 __ATTR(size, 0444, pmem_size_show, NULL); 112 113 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 114 char *buf) 115 { 116 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 117 struct cxl_dev_state *cxlds = cxlmd->cxlds; 118 119 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 120 } 121 static DEVICE_ATTR_RO(serial); 122 123 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 124 char *buf) 125 { 126 return sysfs_emit(buf, "%d\n", dev_to_node(dev)); 127 } 128 static DEVICE_ATTR_RO(numa_node); 129 130 static ssize_t security_state_show(struct device *dev, 131 struct device_attribute *attr, 132 char *buf) 133 { 134 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 135 struct cxl_dev_state *cxlds = cxlmd->cxlds; 136 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 137 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 138 unsigned long state = mds->security.state; 139 int rc = 0; 140 141 /* sync with latest submission state */ 142 mutex_lock(&cxl_mbox->mbox_mutex); 143 if (mds->security.sanitize_active) 144 rc = sysfs_emit(buf, "sanitize\n"); 145 mutex_unlock(&cxl_mbox->mbox_mutex); 146 if (rc) 147 return rc; 148 149 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 150 return sysfs_emit(buf, "disabled\n"); 151 if (state & CXL_PMEM_SEC_STATE_FROZEN || 152 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 153 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 154 return sysfs_emit(buf, "frozen\n"); 155 if (state & CXL_PMEM_SEC_STATE_LOCKED) 156 return sysfs_emit(buf, "locked\n"); 157 158 return sysfs_emit(buf, "unlocked\n"); 159 } 160 static struct device_attribute dev_attr_security_state = 161 __ATTR(state, 0444, security_state_show, NULL); 162 163 static ssize_t security_sanitize_store(struct device *dev, 164 struct device_attribute *attr, 165 const char *buf, size_t len) 166 { 167 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 168 bool sanitize; 169 ssize_t rc; 170 171 if (kstrtobool(buf, &sanitize) || !sanitize) 172 return -EINVAL; 173 174 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 175 if (rc) 176 return rc; 177 178 return len; 179 } 180 static struct device_attribute dev_attr_security_sanitize = 181 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 182 183 static ssize_t security_erase_store(struct device *dev, 184 struct device_attribute *attr, 185 const char *buf, size_t len) 186 { 187 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 188 ssize_t rc; 189 bool erase; 190 191 if (kstrtobool(buf, &erase) || !erase) 192 return -EINVAL; 193 194 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 195 if (rc) 196 return rc; 197 198 return len; 199 } 200 static struct device_attribute dev_attr_security_erase = 201 __ATTR(erase, 0200, NULL, security_erase_store); 202 203 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 204 { 205 struct cxl_dev_state *cxlds = cxlmd->cxlds; 206 u64 offset, length; 207 int rc = 0; 208 209 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 210 for (int i = 0; i < cxlds->nr_partitions; i++) { 211 const struct resource *res = &cxlds->part[i].res; 212 213 offset = res->start; 214 length = resource_size(res); 215 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 216 /* 217 * Invalid Physical Address is not an error for 218 * volatile addresses. Device support is optional. 219 */ 220 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) 221 rc = 0; 222 } 223 return rc; 224 } 225 226 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 227 { 228 struct cxl_port *port; 229 int rc; 230 231 port = cxlmd->endpoint; 232 if (!port || !is_cxl_endpoint(port)) 233 return -EINVAL; 234 235 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); 236 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) 237 return rc; 238 239 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); 240 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) 241 return rc; 242 243 if (cxl_num_decoders_committed(port) == 0) { 244 /* No regions mapped to this memdev */ 245 rc = cxl_get_poison_by_memdev(cxlmd); 246 } else { 247 /* Regions mapped, collect poison by endpoint */ 248 rc = cxl_get_poison_by_endpoint(port); 249 } 250 251 return rc; 252 } 253 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL"); 254 255 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 256 { 257 struct cxl_dev_state *cxlds = cxlmd->cxlds; 258 259 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 260 return 0; 261 262 if (!resource_size(&cxlds->dpa_res)) { 263 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 264 return -EINVAL; 265 } 266 if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) { 267 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 268 dpa, &cxlds->dpa_res); 269 return -EINVAL; 270 } 271 if (!IS_ALIGNED(dpa, 64)) { 272 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 273 return -EINVAL; 274 } 275 276 return 0; 277 } 278 279 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 280 { 281 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 282 struct cxl_mbox_inject_poison inject; 283 struct cxl_poison_record record; 284 struct cxl_mbox_cmd mbox_cmd; 285 struct cxl_region *cxlr; 286 int rc; 287 288 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 289 return 0; 290 291 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); 292 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) 293 return rc; 294 295 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); 296 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) 297 return rc; 298 299 rc = cxl_validate_poison_dpa(cxlmd, dpa); 300 if (rc) 301 return rc; 302 303 inject.address = cpu_to_le64(dpa); 304 mbox_cmd = (struct cxl_mbox_cmd) { 305 .opcode = CXL_MBOX_OP_INJECT_POISON, 306 .size_in = sizeof(inject), 307 .payload_in = &inject, 308 }; 309 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 310 if (rc) 311 return rc; 312 313 cxlr = cxl_dpa_to_region(cxlmd, dpa); 314 if (cxlr) 315 dev_warn_once(cxl_mbox->host, 316 "poison inject dpa:%#llx region: %s\n", dpa, 317 dev_name(&cxlr->dev)); 318 319 record = (struct cxl_poison_record) { 320 .address = cpu_to_le64(dpa), 321 .length = cpu_to_le32(1), 322 }; 323 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 324 325 return 0; 326 } 327 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL"); 328 329 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 330 { 331 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 332 struct cxl_mbox_clear_poison clear; 333 struct cxl_poison_record record; 334 struct cxl_mbox_cmd mbox_cmd; 335 struct cxl_region *cxlr; 336 int rc; 337 338 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 339 return 0; 340 341 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); 342 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) 343 return rc; 344 345 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); 346 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) 347 return rc; 348 349 rc = cxl_validate_poison_dpa(cxlmd, dpa); 350 if (rc) 351 return rc; 352 353 /* 354 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 355 * is defined to accept 64 bytes of write-data, along with the 356 * address to clear. This driver uses zeroes as write-data. 357 */ 358 clear = (struct cxl_mbox_clear_poison) { 359 .address = cpu_to_le64(dpa) 360 }; 361 362 mbox_cmd = (struct cxl_mbox_cmd) { 363 .opcode = CXL_MBOX_OP_CLEAR_POISON, 364 .size_in = sizeof(clear), 365 .payload_in = &clear, 366 }; 367 368 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 369 if (rc) 370 return rc; 371 372 cxlr = cxl_dpa_to_region(cxlmd, dpa); 373 if (cxlr) 374 dev_warn_once(cxl_mbox->host, 375 "poison clear dpa:%#llx region: %s\n", dpa, 376 dev_name(&cxlr->dev)); 377 378 record = (struct cxl_poison_record) { 379 .address = cpu_to_le64(dpa), 380 .length = cpu_to_le32(1), 381 }; 382 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 383 384 return 0; 385 } 386 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL"); 387 388 static struct attribute *cxl_memdev_attributes[] = { 389 &dev_attr_serial.attr, 390 &dev_attr_firmware_version.attr, 391 &dev_attr_payload_max.attr, 392 &dev_attr_label_storage_size.attr, 393 &dev_attr_numa_node.attr, 394 NULL, 395 }; 396 397 static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds) 398 { 399 for (int i = 0; i < cxlds->nr_partitions; i++) 400 if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) 401 return &cxlds->part[i].perf; 402 return NULL; 403 } 404 405 static ssize_t pmem_qos_class_show(struct device *dev, 406 struct device_attribute *attr, char *buf) 407 { 408 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 409 struct cxl_dev_state *cxlds = cxlmd->cxlds; 410 411 return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class); 412 } 413 414 static struct device_attribute dev_attr_pmem_qos_class = 415 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); 416 417 static struct attribute *cxl_memdev_pmem_attributes[] = { 418 &dev_attr_pmem_size.attr, 419 &dev_attr_pmem_qos_class.attr, 420 NULL, 421 }; 422 423 static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds) 424 { 425 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 426 return NULL; 427 return &cxlds->part[0].perf; 428 } 429 430 static ssize_t ram_qos_class_show(struct device *dev, 431 struct device_attribute *attr, char *buf) 432 { 433 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 434 struct cxl_dev_state *cxlds = cxlmd->cxlds; 435 436 return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class); 437 } 438 439 static struct device_attribute dev_attr_ram_qos_class = 440 __ATTR(qos_class, 0444, ram_qos_class_show, NULL); 441 442 static struct attribute *cxl_memdev_ram_attributes[] = { 443 &dev_attr_ram_size.attr, 444 &dev_attr_ram_qos_class.attr, 445 NULL, 446 }; 447 448 static struct attribute *cxl_memdev_security_attributes[] = { 449 &dev_attr_security_state.attr, 450 &dev_attr_security_sanitize.attr, 451 &dev_attr_security_erase.attr, 452 NULL, 453 }; 454 455 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 456 int n) 457 { 458 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 459 return 0; 460 return a->mode; 461 } 462 463 static struct attribute_group cxl_memdev_attribute_group = { 464 .attrs = cxl_memdev_attributes, 465 .is_visible = cxl_memdev_visible, 466 }; 467 468 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) 469 { 470 struct device *dev = kobj_to_dev(kobj); 471 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 472 struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds); 473 474 if (a == &dev_attr_ram_qos_class.attr && 475 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 476 return 0; 477 478 return a->mode; 479 } 480 481 static struct attribute_group cxl_memdev_ram_attribute_group = { 482 .name = "ram", 483 .attrs = cxl_memdev_ram_attributes, 484 .is_visible = cxl_ram_visible, 485 }; 486 487 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) 488 { 489 struct device *dev = kobj_to_dev(kobj); 490 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 491 struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds); 492 493 if (a == &dev_attr_pmem_qos_class.attr && 494 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 495 return 0; 496 497 return a->mode; 498 } 499 500 static struct attribute_group cxl_memdev_pmem_attribute_group = { 501 .name = "pmem", 502 .attrs = cxl_memdev_pmem_attributes, 503 .is_visible = cxl_pmem_visible, 504 }; 505 506 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 507 struct attribute *a, int n) 508 { 509 struct device *dev = kobj_to_dev(kobj); 510 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 511 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 512 513 if (a == &dev_attr_security_sanitize.attr && 514 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 515 return 0; 516 517 if (a == &dev_attr_security_erase.attr && 518 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 519 return 0; 520 521 return a->mode; 522 } 523 524 static struct attribute_group cxl_memdev_security_attribute_group = { 525 .name = "security", 526 .attrs = cxl_memdev_security_attributes, 527 .is_visible = cxl_memdev_security_visible, 528 }; 529 530 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 531 &cxl_memdev_attribute_group, 532 &cxl_memdev_ram_attribute_group, 533 &cxl_memdev_pmem_attribute_group, 534 &cxl_memdev_security_attribute_group, 535 NULL, 536 }; 537 538 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) 539 { 540 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); 541 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); 542 } 543 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL"); 544 545 static const struct device_type cxl_memdev_type = { 546 .name = "cxl_memdev", 547 .release = cxl_memdev_release, 548 .devnode = cxl_memdev_devnode, 549 .groups = cxl_memdev_attribute_groups, 550 }; 551 552 bool is_cxl_memdev(const struct device *dev) 553 { 554 return dev->type == &cxl_memdev_type; 555 } 556 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); 557 558 /** 559 * set_exclusive_cxl_commands() - atomically disable user cxl commands 560 * @mds: The device state to operate on 561 * @cmds: bitmap of commands to mark exclusive 562 * 563 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 564 * invocations of the ioctl path and then disable future execution of 565 * commands with the command ids set in @cmds. 566 */ 567 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 568 unsigned long *cmds) 569 { 570 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 571 572 guard(rwsem_write)(&cxl_memdev_rwsem); 573 bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 574 cmds, CXL_MEM_COMMAND_ID_MAX); 575 } 576 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL"); 577 578 /** 579 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 580 * @mds: The device state to modify 581 * @cmds: bitmap of commands to mark available for userspace 582 */ 583 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 584 unsigned long *cmds) 585 { 586 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 587 588 guard(rwsem_write)(&cxl_memdev_rwsem); 589 bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 590 cmds, CXL_MEM_COMMAND_ID_MAX); 591 } 592 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL"); 593 594 static void cxl_memdev_shutdown(struct device *dev) 595 { 596 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 597 598 guard(rwsem_write)(&cxl_memdev_rwsem); 599 cxlmd->cxlds = NULL; 600 } 601 602 static void cxl_memdev_unregister(void *_cxlmd) 603 { 604 struct cxl_memdev *cxlmd = _cxlmd; 605 struct device *dev = &cxlmd->dev; 606 607 cdev_device_del(&cxlmd->cdev, dev); 608 cxl_memdev_shutdown(dev); 609 put_device(dev); 610 } 611 612 static void detach_memdev(struct work_struct *work) 613 { 614 struct cxl_memdev *cxlmd; 615 616 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 617 device_release_driver(&cxlmd->dev); 618 put_device(&cxlmd->dev); 619 } 620 621 static struct lock_class_key cxl_memdev_key; 622 623 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 624 const struct file_operations *fops) 625 { 626 struct cxl_memdev *cxlmd; 627 struct device *dev; 628 struct cdev *cdev; 629 int rc; 630 631 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 632 if (!cxlmd) 633 return ERR_PTR(-ENOMEM); 634 635 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 636 if (rc < 0) 637 goto err; 638 cxlmd->id = rc; 639 cxlmd->depth = -1; 640 641 dev = &cxlmd->dev; 642 device_initialize(dev); 643 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 644 dev->parent = cxlds->dev; 645 dev->bus = &cxl_bus_type; 646 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 647 dev->type = &cxl_memdev_type; 648 device_set_pm_not_required(dev); 649 INIT_WORK(&cxlmd->detach_work, detach_memdev); 650 651 cdev = &cxlmd->cdev; 652 cdev_init(cdev, fops); 653 return cxlmd; 654 655 err: 656 kfree(cxlmd); 657 return ERR_PTR(rc); 658 } 659 660 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 661 unsigned long arg) 662 { 663 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 664 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 665 666 switch (cmd) { 667 case CXL_MEM_QUERY_COMMANDS: 668 return cxl_query_cmd(cxl_mbox, (void __user *)arg); 669 case CXL_MEM_SEND_COMMAND: 670 return cxl_send_cmd(cxl_mbox, (void __user *)arg); 671 default: 672 return -ENOTTY; 673 } 674 } 675 676 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 677 unsigned long arg) 678 { 679 struct cxl_memdev *cxlmd = file->private_data; 680 struct cxl_dev_state *cxlds; 681 682 guard(rwsem_read)(&cxl_memdev_rwsem); 683 cxlds = cxlmd->cxlds; 684 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 685 return __cxl_memdev_ioctl(cxlmd, cmd, arg); 686 687 return -ENXIO; 688 } 689 690 static int cxl_memdev_open(struct inode *inode, struct file *file) 691 { 692 struct cxl_memdev *cxlmd = 693 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 694 695 get_device(&cxlmd->dev); 696 file->private_data = cxlmd; 697 698 return 0; 699 } 700 701 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 702 { 703 struct cxl_memdev *cxlmd = 704 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 705 706 put_device(&cxlmd->dev); 707 708 return 0; 709 } 710 711 /** 712 * cxl_mem_get_fw_info - Get Firmware info 713 * @mds: The device data for the operation 714 * 715 * Retrieve firmware info for the device specified. 716 * 717 * Return: 0 if no error: or the result of the mailbox command. 718 * 719 * See CXL-3.0 8.2.9.3.1 Get FW Info 720 */ 721 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 722 { 723 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 724 struct cxl_mbox_get_fw_info info; 725 struct cxl_mbox_cmd mbox_cmd; 726 int rc; 727 728 mbox_cmd = (struct cxl_mbox_cmd) { 729 .opcode = CXL_MBOX_OP_GET_FW_INFO, 730 .size_out = sizeof(info), 731 .payload_out = &info, 732 }; 733 734 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 735 if (rc < 0) 736 return rc; 737 738 mds->fw.num_slots = info.num_slots; 739 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 740 info.slot_info); 741 742 return 0; 743 } 744 745 /** 746 * cxl_mem_activate_fw - Activate Firmware 747 * @mds: The device data for the operation 748 * @slot: slot number to activate 749 * 750 * Activate firmware in a given slot for the device specified. 751 * 752 * Return: 0 if no error: or the result of the mailbox command. 753 * 754 * See CXL-3.0 8.2.9.3.3 Activate FW 755 */ 756 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 757 { 758 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 759 struct cxl_mbox_activate_fw activate; 760 struct cxl_mbox_cmd mbox_cmd; 761 762 if (slot == 0 || slot > mds->fw.num_slots) 763 return -EINVAL; 764 765 mbox_cmd = (struct cxl_mbox_cmd) { 766 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 767 .size_in = sizeof(activate), 768 .payload_in = &activate, 769 }; 770 771 /* Only offline activation supported for now */ 772 activate.action = CXL_FW_ACTIVATE_OFFLINE; 773 activate.slot = slot; 774 775 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 776 } 777 778 /** 779 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 780 * @mds: The device data for the operation 781 * 782 * Abort an in-progress firmware transfer for the device specified. 783 * 784 * Return: 0 if no error: or the result of the mailbox command. 785 * 786 * See CXL-3.0 8.2.9.3.2 Transfer FW 787 */ 788 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 789 { 790 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 791 struct cxl_mbox_transfer_fw *transfer; 792 struct cxl_mbox_cmd mbox_cmd; 793 int rc; 794 795 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 796 if (!transfer) 797 return -ENOMEM; 798 799 /* Set a 1s poll interval and a total wait time of 30s */ 800 mbox_cmd = (struct cxl_mbox_cmd) { 801 .opcode = CXL_MBOX_OP_TRANSFER_FW, 802 .size_in = sizeof(*transfer), 803 .payload_in = transfer, 804 .poll_interval_ms = 1000, 805 .poll_count = 30, 806 }; 807 808 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 809 810 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 811 kfree(transfer); 812 return rc; 813 } 814 815 static void cxl_fw_cleanup(struct fw_upload *fwl) 816 { 817 struct cxl_memdev_state *mds = fwl->dd_handle; 818 819 mds->fw.next_slot = 0; 820 } 821 822 static int cxl_fw_do_cancel(struct fw_upload *fwl) 823 { 824 struct cxl_memdev_state *mds = fwl->dd_handle; 825 struct cxl_dev_state *cxlds = &mds->cxlds; 826 struct cxl_memdev *cxlmd = cxlds->cxlmd; 827 int rc; 828 829 rc = cxl_mem_abort_fw_xfer(mds); 830 if (rc < 0) 831 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 832 833 return FW_UPLOAD_ERR_CANCELED; 834 } 835 836 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 837 u32 size) 838 { 839 struct cxl_memdev_state *mds = fwl->dd_handle; 840 struct cxl_mbox_transfer_fw *transfer; 841 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 842 843 if (!size) 844 return FW_UPLOAD_ERR_INVALID_SIZE; 845 846 mds->fw.oneshot = struct_size(transfer, data, size) < 847 cxl_mbox->payload_size; 848 849 if (cxl_mem_get_fw_info(mds)) 850 return FW_UPLOAD_ERR_HW_ERROR; 851 852 /* 853 * So far no state has been changed, hence no other cleanup is 854 * necessary. Simply return the cancelled status. 855 */ 856 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 857 return FW_UPLOAD_ERR_CANCELED; 858 859 return FW_UPLOAD_ERR_NONE; 860 } 861 862 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 863 u32 offset, u32 size, u32 *written) 864 { 865 struct cxl_memdev_state *mds = fwl->dd_handle; 866 struct cxl_dev_state *cxlds = &mds->cxlds; 867 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 868 struct cxl_memdev *cxlmd = cxlds->cxlmd; 869 struct cxl_mbox_transfer_fw *transfer; 870 struct cxl_mbox_cmd mbox_cmd; 871 u32 cur_size, remaining; 872 size_t size_in; 873 int rc; 874 875 *written = 0; 876 877 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 878 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 879 dev_err(&cxlmd->dev, 880 "misaligned offset for FW transfer slice (%u)\n", 881 offset); 882 return FW_UPLOAD_ERR_RW_ERROR; 883 } 884 885 /* 886 * Pick transfer size based on mds->payload_size @size must bw 128-byte 887 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 888 * sizeof(*transfer) is 128. These constraints imply that @cur_size 889 * will always be 128b aligned. 890 */ 891 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer)); 892 893 remaining = size - cur_size; 894 size_in = struct_size(transfer, data, cur_size); 895 896 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 897 return cxl_fw_do_cancel(fwl); 898 899 /* 900 * Slot numbers are 1-indexed 901 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 902 * Check for rollover using modulo, and 1-index it by adding 1 903 */ 904 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 905 906 /* Do the transfer via mailbox cmd */ 907 transfer = kzalloc(size_in, GFP_KERNEL); 908 if (!transfer) 909 return FW_UPLOAD_ERR_RW_ERROR; 910 911 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 912 memcpy(transfer->data, data + offset, cur_size); 913 if (mds->fw.oneshot) { 914 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 915 transfer->slot = mds->fw.next_slot; 916 } else { 917 if (offset == 0) { 918 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 919 } else if (remaining == 0) { 920 transfer->action = CXL_FW_TRANSFER_ACTION_END; 921 transfer->slot = mds->fw.next_slot; 922 } else { 923 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 924 } 925 } 926 927 mbox_cmd = (struct cxl_mbox_cmd) { 928 .opcode = CXL_MBOX_OP_TRANSFER_FW, 929 .size_in = size_in, 930 .payload_in = transfer, 931 .poll_interval_ms = 1000, 932 .poll_count = 30, 933 }; 934 935 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 936 if (rc < 0) { 937 rc = FW_UPLOAD_ERR_RW_ERROR; 938 goto out_free; 939 } 940 941 *written = cur_size; 942 943 /* Activate FW if oneshot or if the last slice was written */ 944 if (mds->fw.oneshot || remaining == 0) { 945 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 946 mds->fw.next_slot); 947 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 948 if (rc < 0) { 949 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 950 rc); 951 rc = FW_UPLOAD_ERR_HW_ERROR; 952 goto out_free; 953 } 954 } 955 956 rc = FW_UPLOAD_ERR_NONE; 957 958 out_free: 959 kfree(transfer); 960 return rc; 961 } 962 963 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 964 { 965 struct cxl_memdev_state *mds = fwl->dd_handle; 966 967 /* 968 * cxl_internal_send_cmd() handles background operations synchronously. 969 * No need to wait for completions here - any errors would've been 970 * reported and handled during the ->write() call(s). 971 * Just check if a cancel request was received, and return success. 972 */ 973 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 974 return cxl_fw_do_cancel(fwl); 975 976 return FW_UPLOAD_ERR_NONE; 977 } 978 979 static void cxl_fw_cancel(struct fw_upload *fwl) 980 { 981 struct cxl_memdev_state *mds = fwl->dd_handle; 982 983 set_bit(CXL_FW_CANCEL, mds->fw.state); 984 } 985 986 static const struct fw_upload_ops cxl_memdev_fw_ops = { 987 .prepare = cxl_fw_prepare, 988 .write = cxl_fw_write, 989 .poll_complete = cxl_fw_poll_complete, 990 .cancel = cxl_fw_cancel, 991 .cleanup = cxl_fw_cleanup, 992 }; 993 994 static void cxl_remove_fw_upload(void *fwl) 995 { 996 firmware_upload_unregister(fwl); 997 } 998 999 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 1000 { 1001 struct cxl_dev_state *cxlds = &mds->cxlds; 1002 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 1003 struct device *dev = &cxlds->cxlmd->dev; 1004 struct fw_upload *fwl; 1005 1006 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds)) 1007 return 0; 1008 1009 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 1010 &cxl_memdev_fw_ops, mds); 1011 if (IS_ERR(fwl)) 1012 return PTR_ERR(fwl); 1013 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 1014 } 1015 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL"); 1016 1017 static const struct file_operations cxl_memdev_fops = { 1018 .owner = THIS_MODULE, 1019 .unlocked_ioctl = cxl_memdev_ioctl, 1020 .open = cxl_memdev_open, 1021 .release = cxl_memdev_release_file, 1022 .compat_ioctl = compat_ptr_ioctl, 1023 .llseek = noop_llseek, 1024 }; 1025 1026 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 1027 struct cxl_dev_state *cxlds) 1028 { 1029 struct cxl_memdev *cxlmd; 1030 struct device *dev; 1031 struct cdev *cdev; 1032 int rc; 1033 1034 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1035 if (IS_ERR(cxlmd)) 1036 return cxlmd; 1037 1038 dev = &cxlmd->dev; 1039 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1040 if (rc) 1041 goto err; 1042 1043 /* 1044 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1045 * needed as this is ordered with cdev_add() publishing the device. 1046 */ 1047 cxlmd->cxlds = cxlds; 1048 cxlds->cxlmd = cxlmd; 1049 1050 cdev = &cxlmd->cdev; 1051 rc = cdev_device_add(cdev, dev); 1052 if (rc) 1053 goto err; 1054 1055 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1056 if (rc) 1057 return ERR_PTR(rc); 1058 return cxlmd; 1059 1060 err: 1061 /* 1062 * The cdev was briefly live, shutdown any ioctl operations that 1063 * saw that state. 1064 */ 1065 cxl_memdev_shutdown(dev); 1066 put_device(dev); 1067 return ERR_PTR(rc); 1068 } 1069 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL"); 1070 1071 static void sanitize_teardown_notifier(void *data) 1072 { 1073 struct cxl_memdev_state *mds = data; 1074 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 1075 struct kernfs_node *state; 1076 1077 /* 1078 * Prevent new irq triggered invocations of the workqueue and 1079 * flush inflight invocations. 1080 */ 1081 mutex_lock(&cxl_mbox->mbox_mutex); 1082 state = mds->security.sanitize_node; 1083 mds->security.sanitize_node = NULL; 1084 mutex_unlock(&cxl_mbox->mbox_mutex); 1085 1086 cancel_delayed_work_sync(&mds->security.poll_dwork); 1087 sysfs_put(state); 1088 } 1089 1090 int devm_cxl_sanitize_setup_notifier(struct device *host, 1091 struct cxl_memdev *cxlmd) 1092 { 1093 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1094 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1095 struct kernfs_node *sec; 1096 1097 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1098 return 0; 1099 1100 /* 1101 * Note, the expectation is that @cxlmd would have failed to be 1102 * created if these sysfs_get_dirent calls fail. 1103 */ 1104 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1105 if (!sec) 1106 return -ENOENT; 1107 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1108 sysfs_put(sec); 1109 if (!mds->security.sanitize_node) 1110 return -ENOENT; 1111 1112 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1113 } 1114 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL"); 1115 1116 __init int cxl_memdev_init(void) 1117 { 1118 dev_t devt; 1119 int rc; 1120 1121 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1122 if (rc) 1123 return rc; 1124 1125 cxl_mem_major = MAJOR(devt); 1126 1127 return 0; 1128 } 1129 1130 void cxl_memdev_exit(void) 1131 { 1132 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1133 } 1134