1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 devm_cxl_memdev_edac_release(cxlmd); 31 kfree(cxlmd); 32 } 33 34 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 35 kgid_t *gid) 36 { 37 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 38 } 39 40 static ssize_t firmware_version_show(struct device *dev, 41 struct device_attribute *attr, char *buf) 42 { 43 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 44 struct cxl_dev_state *cxlds = cxlmd->cxlds; 45 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 46 47 if (!mds) 48 return sysfs_emit(buf, "\n"); 49 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 50 } 51 static DEVICE_ATTR_RO(firmware_version); 52 53 static ssize_t payload_max_show(struct device *dev, 54 struct device_attribute *attr, char *buf) 55 { 56 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 57 struct cxl_dev_state *cxlds = cxlmd->cxlds; 58 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 59 60 if (!mds) 61 return sysfs_emit(buf, "\n"); 62 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size); 63 } 64 static DEVICE_ATTR_RO(payload_max); 65 66 static ssize_t label_storage_size_show(struct device *dev, 67 struct device_attribute *attr, char *buf) 68 { 69 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 70 struct cxl_dev_state *cxlds = cxlmd->cxlds; 71 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 72 73 if (!mds) 74 return sysfs_emit(buf, "\n"); 75 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 76 } 77 static DEVICE_ATTR_RO(label_storage_size); 78 79 static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds) 80 { 81 /* Static RAM is only expected at partition 0. */ 82 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 83 return 0; 84 return resource_size(&cxlds->part[0].res); 85 } 86 87 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 88 char *buf) 89 { 90 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 91 struct cxl_dev_state *cxlds = cxlmd->cxlds; 92 unsigned long long len = cxl_ram_size(cxlds); 93 94 return sysfs_emit(buf, "%#llx\n", len); 95 } 96 97 static struct device_attribute dev_attr_ram_size = 98 __ATTR(size, 0444, ram_size_show, NULL); 99 100 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 101 char *buf) 102 { 103 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 104 struct cxl_dev_state *cxlds = cxlmd->cxlds; 105 unsigned long long len = cxl_pmem_size(cxlds); 106 107 return sysfs_emit(buf, "%#llx\n", len); 108 } 109 110 static struct device_attribute dev_attr_pmem_size = 111 __ATTR(size, 0444, pmem_size_show, NULL); 112 113 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 114 char *buf) 115 { 116 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 117 struct cxl_dev_state *cxlds = cxlmd->cxlds; 118 119 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 120 } 121 static DEVICE_ATTR_RO(serial); 122 123 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 124 char *buf) 125 { 126 return sysfs_emit(buf, "%d\n", dev_to_node(dev)); 127 } 128 static DEVICE_ATTR_RO(numa_node); 129 130 static ssize_t security_state_show(struct device *dev, 131 struct device_attribute *attr, 132 char *buf) 133 { 134 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 135 struct cxl_dev_state *cxlds = cxlmd->cxlds; 136 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 137 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 138 unsigned long state = mds->security.state; 139 int rc = 0; 140 141 /* sync with latest submission state */ 142 mutex_lock(&cxl_mbox->mbox_mutex); 143 if (mds->security.sanitize_active) 144 rc = sysfs_emit(buf, "sanitize\n"); 145 mutex_unlock(&cxl_mbox->mbox_mutex); 146 if (rc) 147 return rc; 148 149 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 150 return sysfs_emit(buf, "disabled\n"); 151 if (state & CXL_PMEM_SEC_STATE_FROZEN || 152 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 153 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 154 return sysfs_emit(buf, "frozen\n"); 155 if (state & CXL_PMEM_SEC_STATE_LOCKED) 156 return sysfs_emit(buf, "locked\n"); 157 158 return sysfs_emit(buf, "unlocked\n"); 159 } 160 static struct device_attribute dev_attr_security_state = 161 __ATTR(state, 0444, security_state_show, NULL); 162 163 static ssize_t security_sanitize_store(struct device *dev, 164 struct device_attribute *attr, 165 const char *buf, size_t len) 166 { 167 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 168 bool sanitize; 169 ssize_t rc; 170 171 if (kstrtobool(buf, &sanitize) || !sanitize) 172 return -EINVAL; 173 174 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 175 if (rc) 176 return rc; 177 178 return len; 179 } 180 static struct device_attribute dev_attr_security_sanitize = 181 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 182 183 static ssize_t security_erase_store(struct device *dev, 184 struct device_attribute *attr, 185 const char *buf, size_t len) 186 { 187 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 188 ssize_t rc; 189 bool erase; 190 191 if (kstrtobool(buf, &erase) || !erase) 192 return -EINVAL; 193 194 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 195 if (rc) 196 return rc; 197 198 return len; 199 } 200 static struct device_attribute dev_attr_security_erase = 201 __ATTR(erase, 0200, NULL, security_erase_store); 202 203 bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd, 204 enum poison_cmd_enabled_bits cmd) 205 { 206 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 207 208 return test_bit(cmd, mds->poison.enabled_cmds); 209 } 210 211 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 212 { 213 struct cxl_dev_state *cxlds = cxlmd->cxlds; 214 u64 offset, length; 215 int rc = 0; 216 217 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 218 for (int i = 0; i < cxlds->nr_partitions; i++) { 219 const struct resource *res = &cxlds->part[i].res; 220 221 offset = res->start; 222 length = resource_size(res); 223 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 224 /* 225 * Invalid Physical Address is not an error for 226 * volatile addresses. Device support is optional. 227 */ 228 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) 229 rc = 0; 230 } 231 return rc; 232 } 233 234 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 235 { 236 struct cxl_port *port; 237 int rc; 238 239 port = cxlmd->endpoint; 240 if (!port || !is_cxl_endpoint(port)) 241 return -EINVAL; 242 243 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); 244 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) 245 return rc; 246 247 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); 248 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) 249 return rc; 250 251 if (cxl_num_decoders_committed(port) == 0) { 252 /* No regions mapped to this memdev */ 253 rc = cxl_get_poison_by_memdev(cxlmd); 254 } else { 255 /* Regions mapped, collect poison by endpoint */ 256 rc = cxl_get_poison_by_endpoint(port); 257 } 258 259 return rc; 260 } 261 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL"); 262 263 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 264 { 265 struct cxl_dev_state *cxlds = cxlmd->cxlds; 266 267 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 268 return 0; 269 270 if (!resource_size(&cxlds->dpa_res)) { 271 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 272 return -EINVAL; 273 } 274 if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) { 275 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 276 dpa, &cxlds->dpa_res); 277 return -EINVAL; 278 } 279 if (!IS_ALIGNED(dpa, 64)) { 280 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 281 return -EINVAL; 282 } 283 284 return 0; 285 } 286 287 int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa) 288 { 289 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 290 struct cxl_mbox_inject_poison inject; 291 struct cxl_poison_record record; 292 struct cxl_mbox_cmd mbox_cmd; 293 struct cxl_region *cxlr; 294 int rc; 295 296 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 297 return 0; 298 299 lockdep_assert_held(&cxl_rwsem.dpa); 300 lockdep_assert_held(&cxl_rwsem.region); 301 302 rc = cxl_validate_poison_dpa(cxlmd, dpa); 303 if (rc) 304 return rc; 305 306 inject.address = cpu_to_le64(dpa); 307 mbox_cmd = (struct cxl_mbox_cmd) { 308 .opcode = CXL_MBOX_OP_INJECT_POISON, 309 .size_in = sizeof(inject), 310 .payload_in = &inject, 311 }; 312 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 313 if (rc) 314 return rc; 315 316 cxlr = cxl_dpa_to_region(cxlmd, dpa); 317 if (cxlr) 318 dev_warn_once(cxl_mbox->host, 319 "poison inject dpa:%#llx region: %s\n", dpa, 320 dev_name(&cxlr->dev)); 321 322 record = (struct cxl_poison_record) { 323 .address = cpu_to_le64(dpa), 324 .length = cpu_to_le32(1), 325 }; 326 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 327 328 return 0; 329 } 330 331 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 332 { 333 int rc; 334 335 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); 336 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) 337 return rc; 338 339 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); 340 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) 341 return rc; 342 343 return cxl_inject_poison_locked(cxlmd, dpa); 344 } 345 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL"); 346 347 int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa) 348 { 349 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 350 struct cxl_mbox_clear_poison clear; 351 struct cxl_poison_record record; 352 struct cxl_mbox_cmd mbox_cmd; 353 struct cxl_region *cxlr; 354 int rc; 355 356 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 357 return 0; 358 359 lockdep_assert_held(&cxl_rwsem.dpa); 360 lockdep_assert_held(&cxl_rwsem.region); 361 362 rc = cxl_validate_poison_dpa(cxlmd, dpa); 363 if (rc) 364 return rc; 365 366 /* 367 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 368 * is defined to accept 64 bytes of write-data, along with the 369 * address to clear. This driver uses zeroes as write-data. 370 */ 371 clear = (struct cxl_mbox_clear_poison) { 372 .address = cpu_to_le64(dpa) 373 }; 374 375 mbox_cmd = (struct cxl_mbox_cmd) { 376 .opcode = CXL_MBOX_OP_CLEAR_POISON, 377 .size_in = sizeof(clear), 378 .payload_in = &clear, 379 }; 380 381 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 382 if (rc) 383 return rc; 384 385 cxlr = cxl_dpa_to_region(cxlmd, dpa); 386 if (cxlr) 387 dev_warn_once(cxl_mbox->host, 388 "poison clear dpa:%#llx region: %s\n", dpa, 389 dev_name(&cxlr->dev)); 390 391 record = (struct cxl_poison_record) { 392 .address = cpu_to_le64(dpa), 393 .length = cpu_to_le32(1), 394 }; 395 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 396 397 return 0; 398 } 399 400 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 401 { 402 int rc; 403 404 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); 405 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) 406 return rc; 407 408 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); 409 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) 410 return rc; 411 412 return cxl_clear_poison_locked(cxlmd, dpa); 413 } 414 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL"); 415 416 static struct attribute *cxl_memdev_attributes[] = { 417 &dev_attr_serial.attr, 418 &dev_attr_firmware_version.attr, 419 &dev_attr_payload_max.attr, 420 &dev_attr_label_storage_size.attr, 421 &dev_attr_numa_node.attr, 422 NULL, 423 }; 424 425 static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds) 426 { 427 for (int i = 0; i < cxlds->nr_partitions; i++) 428 if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) 429 return &cxlds->part[i].perf; 430 return NULL; 431 } 432 433 static ssize_t pmem_qos_class_show(struct device *dev, 434 struct device_attribute *attr, char *buf) 435 { 436 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 437 struct cxl_dev_state *cxlds = cxlmd->cxlds; 438 439 return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class); 440 } 441 442 static struct device_attribute dev_attr_pmem_qos_class = 443 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); 444 445 static struct attribute *cxl_memdev_pmem_attributes[] = { 446 &dev_attr_pmem_size.attr, 447 &dev_attr_pmem_qos_class.attr, 448 NULL, 449 }; 450 451 static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds) 452 { 453 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 454 return NULL; 455 return &cxlds->part[0].perf; 456 } 457 458 static ssize_t ram_qos_class_show(struct device *dev, 459 struct device_attribute *attr, char *buf) 460 { 461 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 462 struct cxl_dev_state *cxlds = cxlmd->cxlds; 463 464 return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class); 465 } 466 467 static struct device_attribute dev_attr_ram_qos_class = 468 __ATTR(qos_class, 0444, ram_qos_class_show, NULL); 469 470 static struct attribute *cxl_memdev_ram_attributes[] = { 471 &dev_attr_ram_size.attr, 472 &dev_attr_ram_qos_class.attr, 473 NULL, 474 }; 475 476 static struct attribute *cxl_memdev_security_attributes[] = { 477 &dev_attr_security_state.attr, 478 &dev_attr_security_sanitize.attr, 479 &dev_attr_security_erase.attr, 480 NULL, 481 }; 482 483 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 484 int n) 485 { 486 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 487 return 0; 488 return a->mode; 489 } 490 491 static struct attribute_group cxl_memdev_attribute_group = { 492 .attrs = cxl_memdev_attributes, 493 .is_visible = cxl_memdev_visible, 494 }; 495 496 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) 497 { 498 struct device *dev = kobj_to_dev(kobj); 499 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 500 struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds); 501 502 if (a == &dev_attr_ram_qos_class.attr && 503 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 504 return 0; 505 506 return a->mode; 507 } 508 509 static struct attribute_group cxl_memdev_ram_attribute_group = { 510 .name = "ram", 511 .attrs = cxl_memdev_ram_attributes, 512 .is_visible = cxl_ram_visible, 513 }; 514 515 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) 516 { 517 struct device *dev = kobj_to_dev(kobj); 518 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 519 struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds); 520 521 if (a == &dev_attr_pmem_qos_class.attr && 522 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 523 return 0; 524 525 return a->mode; 526 } 527 528 static struct attribute_group cxl_memdev_pmem_attribute_group = { 529 .name = "pmem", 530 .attrs = cxl_memdev_pmem_attributes, 531 .is_visible = cxl_pmem_visible, 532 }; 533 534 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 535 struct attribute *a, int n) 536 { 537 struct device *dev = kobj_to_dev(kobj); 538 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 539 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 540 541 if (a == &dev_attr_security_sanitize.attr && 542 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 543 return 0; 544 545 if (a == &dev_attr_security_erase.attr && 546 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 547 return 0; 548 549 return a->mode; 550 } 551 552 static struct attribute_group cxl_memdev_security_attribute_group = { 553 .name = "security", 554 .attrs = cxl_memdev_security_attributes, 555 .is_visible = cxl_memdev_security_visible, 556 }; 557 558 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 559 &cxl_memdev_attribute_group, 560 &cxl_memdev_ram_attribute_group, 561 &cxl_memdev_pmem_attribute_group, 562 &cxl_memdev_security_attribute_group, 563 NULL, 564 }; 565 566 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) 567 { 568 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); 569 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); 570 } 571 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL"); 572 573 static const struct device_type cxl_memdev_type = { 574 .name = "cxl_memdev", 575 .release = cxl_memdev_release, 576 .devnode = cxl_memdev_devnode, 577 .groups = cxl_memdev_attribute_groups, 578 }; 579 580 bool is_cxl_memdev(const struct device *dev) 581 { 582 return dev->type == &cxl_memdev_type; 583 } 584 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); 585 586 /** 587 * set_exclusive_cxl_commands() - atomically disable user cxl commands 588 * @mds: The device state to operate on 589 * @cmds: bitmap of commands to mark exclusive 590 * 591 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 592 * invocations of the ioctl path and then disable future execution of 593 * commands with the command ids set in @cmds. 594 */ 595 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 596 unsigned long *cmds) 597 { 598 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 599 600 guard(rwsem_write)(&cxl_memdev_rwsem); 601 bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 602 cmds, CXL_MEM_COMMAND_ID_MAX); 603 } 604 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL"); 605 606 /** 607 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 608 * @mds: The device state to modify 609 * @cmds: bitmap of commands to mark available for userspace 610 */ 611 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 612 unsigned long *cmds) 613 { 614 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 615 616 guard(rwsem_write)(&cxl_memdev_rwsem); 617 bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 618 cmds, CXL_MEM_COMMAND_ID_MAX); 619 } 620 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL"); 621 622 static void cxl_memdev_shutdown(struct device *dev) 623 { 624 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 625 626 guard(rwsem_write)(&cxl_memdev_rwsem); 627 cxlmd->cxlds = NULL; 628 } 629 630 static void cxl_memdev_unregister(void *_cxlmd) 631 { 632 struct cxl_memdev *cxlmd = _cxlmd; 633 struct device *dev = &cxlmd->dev; 634 635 cdev_device_del(&cxlmd->cdev, dev); 636 cxl_memdev_shutdown(dev); 637 put_device(dev); 638 } 639 640 static void detach_memdev(struct work_struct *work) 641 { 642 struct cxl_memdev *cxlmd; 643 644 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 645 device_release_driver(&cxlmd->dev); 646 put_device(&cxlmd->dev); 647 } 648 649 static struct lock_class_key cxl_memdev_key; 650 651 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 652 const struct file_operations *fops) 653 { 654 struct cxl_memdev *cxlmd; 655 struct device *dev; 656 struct cdev *cdev; 657 int rc; 658 659 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 660 if (!cxlmd) 661 return ERR_PTR(-ENOMEM); 662 663 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 664 if (rc < 0) 665 goto err; 666 cxlmd->id = rc; 667 cxlmd->depth = -1; 668 669 dev = &cxlmd->dev; 670 device_initialize(dev); 671 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 672 dev->parent = cxlds->dev; 673 dev->bus = &cxl_bus_type; 674 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 675 dev->type = &cxl_memdev_type; 676 device_set_pm_not_required(dev); 677 INIT_WORK(&cxlmd->detach_work, detach_memdev); 678 679 cdev = &cxlmd->cdev; 680 cdev_init(cdev, fops); 681 return cxlmd; 682 683 err: 684 kfree(cxlmd); 685 return ERR_PTR(rc); 686 } 687 688 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 689 unsigned long arg) 690 { 691 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 692 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 693 694 switch (cmd) { 695 case CXL_MEM_QUERY_COMMANDS: 696 return cxl_query_cmd(cxl_mbox, (void __user *)arg); 697 case CXL_MEM_SEND_COMMAND: 698 return cxl_send_cmd(cxl_mbox, (void __user *)arg); 699 default: 700 return -ENOTTY; 701 } 702 } 703 704 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 705 unsigned long arg) 706 { 707 struct cxl_memdev *cxlmd = file->private_data; 708 struct cxl_dev_state *cxlds; 709 710 guard(rwsem_read)(&cxl_memdev_rwsem); 711 cxlds = cxlmd->cxlds; 712 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 713 return __cxl_memdev_ioctl(cxlmd, cmd, arg); 714 715 return -ENXIO; 716 } 717 718 static int cxl_memdev_open(struct inode *inode, struct file *file) 719 { 720 struct cxl_memdev *cxlmd = 721 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 722 723 get_device(&cxlmd->dev); 724 file->private_data = cxlmd; 725 726 return 0; 727 } 728 729 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 730 { 731 struct cxl_memdev *cxlmd = 732 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 733 734 put_device(&cxlmd->dev); 735 736 return 0; 737 } 738 739 /** 740 * cxl_mem_get_fw_info - Get Firmware info 741 * @mds: The device data for the operation 742 * 743 * Retrieve firmware info for the device specified. 744 * 745 * Return: 0 if no error: or the result of the mailbox command. 746 * 747 * See CXL-3.0 8.2.9.3.1 Get FW Info 748 */ 749 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 750 { 751 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 752 struct cxl_mbox_get_fw_info info; 753 struct cxl_mbox_cmd mbox_cmd; 754 int rc; 755 756 mbox_cmd = (struct cxl_mbox_cmd) { 757 .opcode = CXL_MBOX_OP_GET_FW_INFO, 758 .size_out = sizeof(info), 759 .payload_out = &info, 760 }; 761 762 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 763 if (rc < 0) 764 return rc; 765 766 mds->fw.num_slots = info.num_slots; 767 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 768 info.slot_info); 769 770 return 0; 771 } 772 773 /** 774 * cxl_mem_activate_fw - Activate Firmware 775 * @mds: The device data for the operation 776 * @slot: slot number to activate 777 * 778 * Activate firmware in a given slot for the device specified. 779 * 780 * Return: 0 if no error: or the result of the mailbox command. 781 * 782 * See CXL-3.0 8.2.9.3.3 Activate FW 783 */ 784 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 785 { 786 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 787 struct cxl_mbox_activate_fw activate; 788 struct cxl_mbox_cmd mbox_cmd; 789 790 if (slot == 0 || slot > mds->fw.num_slots) 791 return -EINVAL; 792 793 mbox_cmd = (struct cxl_mbox_cmd) { 794 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 795 .size_in = sizeof(activate), 796 .payload_in = &activate, 797 }; 798 799 /* Only offline activation supported for now */ 800 activate.action = CXL_FW_ACTIVATE_OFFLINE; 801 activate.slot = slot; 802 803 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 804 } 805 806 /** 807 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 808 * @mds: The device data for the operation 809 * 810 * Abort an in-progress firmware transfer for the device specified. 811 * 812 * Return: 0 if no error: or the result of the mailbox command. 813 * 814 * See CXL-3.0 8.2.9.3.2 Transfer FW 815 */ 816 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 817 { 818 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 819 struct cxl_mbox_transfer_fw *transfer; 820 struct cxl_mbox_cmd mbox_cmd; 821 int rc; 822 823 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 824 if (!transfer) 825 return -ENOMEM; 826 827 /* Set a 1s poll interval and a total wait time of 30s */ 828 mbox_cmd = (struct cxl_mbox_cmd) { 829 .opcode = CXL_MBOX_OP_TRANSFER_FW, 830 .size_in = sizeof(*transfer), 831 .payload_in = transfer, 832 .poll_interval_ms = 1000, 833 .poll_count = 30, 834 }; 835 836 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 837 838 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 839 kfree(transfer); 840 return rc; 841 } 842 843 static void cxl_fw_cleanup(struct fw_upload *fwl) 844 { 845 struct cxl_memdev_state *mds = fwl->dd_handle; 846 847 mds->fw.next_slot = 0; 848 } 849 850 static int cxl_fw_do_cancel(struct fw_upload *fwl) 851 { 852 struct cxl_memdev_state *mds = fwl->dd_handle; 853 struct cxl_dev_state *cxlds = &mds->cxlds; 854 struct cxl_memdev *cxlmd = cxlds->cxlmd; 855 int rc; 856 857 rc = cxl_mem_abort_fw_xfer(mds); 858 if (rc < 0) 859 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 860 861 return FW_UPLOAD_ERR_CANCELED; 862 } 863 864 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 865 u32 size) 866 { 867 struct cxl_memdev_state *mds = fwl->dd_handle; 868 struct cxl_mbox_transfer_fw *transfer; 869 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 870 871 if (!size) 872 return FW_UPLOAD_ERR_INVALID_SIZE; 873 874 mds->fw.oneshot = struct_size(transfer, data, size) < 875 cxl_mbox->payload_size; 876 877 if (cxl_mem_get_fw_info(mds)) 878 return FW_UPLOAD_ERR_HW_ERROR; 879 880 /* 881 * So far no state has been changed, hence no other cleanup is 882 * necessary. Simply return the cancelled status. 883 */ 884 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 885 return FW_UPLOAD_ERR_CANCELED; 886 887 return FW_UPLOAD_ERR_NONE; 888 } 889 890 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 891 u32 offset, u32 size, u32 *written) 892 { 893 struct cxl_memdev_state *mds = fwl->dd_handle; 894 struct cxl_dev_state *cxlds = &mds->cxlds; 895 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 896 struct cxl_memdev *cxlmd = cxlds->cxlmd; 897 struct cxl_mbox_transfer_fw *transfer; 898 struct cxl_mbox_cmd mbox_cmd; 899 u32 cur_size, remaining; 900 size_t size_in; 901 int rc; 902 903 *written = 0; 904 905 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 906 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 907 dev_err(&cxlmd->dev, 908 "misaligned offset for FW transfer slice (%u)\n", 909 offset); 910 return FW_UPLOAD_ERR_RW_ERROR; 911 } 912 913 /* 914 * Pick transfer size based on mds->payload_size @size must bw 128-byte 915 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 916 * sizeof(*transfer) is 128. These constraints imply that @cur_size 917 * will always be 128b aligned. 918 */ 919 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer)); 920 921 remaining = size - cur_size; 922 size_in = struct_size(transfer, data, cur_size); 923 924 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 925 return cxl_fw_do_cancel(fwl); 926 927 /* 928 * Slot numbers are 1-indexed 929 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 930 * Check for rollover using modulo, and 1-index it by adding 1 931 */ 932 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 933 934 /* Do the transfer via mailbox cmd */ 935 transfer = kzalloc(size_in, GFP_KERNEL); 936 if (!transfer) 937 return FW_UPLOAD_ERR_RW_ERROR; 938 939 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 940 memcpy(transfer->data, data + offset, cur_size); 941 if (mds->fw.oneshot) { 942 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 943 transfer->slot = mds->fw.next_slot; 944 } else { 945 if (offset == 0) { 946 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 947 } else if (remaining == 0) { 948 transfer->action = CXL_FW_TRANSFER_ACTION_END; 949 transfer->slot = mds->fw.next_slot; 950 } else { 951 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 952 } 953 } 954 955 mbox_cmd = (struct cxl_mbox_cmd) { 956 .opcode = CXL_MBOX_OP_TRANSFER_FW, 957 .size_in = size_in, 958 .payload_in = transfer, 959 .poll_interval_ms = 1000, 960 .poll_count = 30, 961 }; 962 963 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 964 if (rc < 0) { 965 rc = FW_UPLOAD_ERR_RW_ERROR; 966 goto out_free; 967 } 968 969 *written = cur_size; 970 971 /* Activate FW if oneshot or if the last slice was written */ 972 if (mds->fw.oneshot || remaining == 0) { 973 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 974 mds->fw.next_slot); 975 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 976 if (rc < 0) { 977 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 978 rc); 979 rc = FW_UPLOAD_ERR_HW_ERROR; 980 goto out_free; 981 } 982 } 983 984 rc = FW_UPLOAD_ERR_NONE; 985 986 out_free: 987 kfree(transfer); 988 return rc; 989 } 990 991 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 992 { 993 struct cxl_memdev_state *mds = fwl->dd_handle; 994 995 /* 996 * cxl_internal_send_cmd() handles background operations synchronously. 997 * No need to wait for completions here - any errors would've been 998 * reported and handled during the ->write() call(s). 999 * Just check if a cancel request was received, and return success. 1000 */ 1001 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 1002 return cxl_fw_do_cancel(fwl); 1003 1004 return FW_UPLOAD_ERR_NONE; 1005 } 1006 1007 static void cxl_fw_cancel(struct fw_upload *fwl) 1008 { 1009 struct cxl_memdev_state *mds = fwl->dd_handle; 1010 1011 set_bit(CXL_FW_CANCEL, mds->fw.state); 1012 } 1013 1014 static const struct fw_upload_ops cxl_memdev_fw_ops = { 1015 .prepare = cxl_fw_prepare, 1016 .write = cxl_fw_write, 1017 .poll_complete = cxl_fw_poll_complete, 1018 .cancel = cxl_fw_cancel, 1019 .cleanup = cxl_fw_cleanup, 1020 }; 1021 1022 static void cxl_remove_fw_upload(void *fwl) 1023 { 1024 firmware_upload_unregister(fwl); 1025 } 1026 1027 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 1028 { 1029 struct cxl_dev_state *cxlds = &mds->cxlds; 1030 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 1031 struct device *dev = &cxlds->cxlmd->dev; 1032 struct fw_upload *fwl; 1033 1034 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds)) 1035 return 0; 1036 1037 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 1038 &cxl_memdev_fw_ops, mds); 1039 if (IS_ERR(fwl)) 1040 return PTR_ERR(fwl); 1041 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 1042 } 1043 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL"); 1044 1045 static const struct file_operations cxl_memdev_fops = { 1046 .owner = THIS_MODULE, 1047 .unlocked_ioctl = cxl_memdev_ioctl, 1048 .open = cxl_memdev_open, 1049 .release = cxl_memdev_release_file, 1050 .compat_ioctl = compat_ptr_ioctl, 1051 .llseek = noop_llseek, 1052 }; 1053 1054 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 1055 struct cxl_dev_state *cxlds) 1056 { 1057 struct cxl_memdev *cxlmd; 1058 struct device *dev; 1059 struct cdev *cdev; 1060 int rc; 1061 1062 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1063 if (IS_ERR(cxlmd)) 1064 return cxlmd; 1065 1066 dev = &cxlmd->dev; 1067 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1068 if (rc) 1069 goto err; 1070 1071 /* 1072 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1073 * needed as this is ordered with cdev_add() publishing the device. 1074 */ 1075 cxlmd->cxlds = cxlds; 1076 cxlds->cxlmd = cxlmd; 1077 1078 cdev = &cxlmd->cdev; 1079 rc = cdev_device_add(cdev, dev); 1080 if (rc) 1081 goto err; 1082 1083 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1084 if (rc) 1085 return ERR_PTR(rc); 1086 return cxlmd; 1087 1088 err: 1089 /* 1090 * The cdev was briefly live, shutdown any ioctl operations that 1091 * saw that state. 1092 */ 1093 cxl_memdev_shutdown(dev); 1094 put_device(dev); 1095 return ERR_PTR(rc); 1096 } 1097 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL"); 1098 1099 static void sanitize_teardown_notifier(void *data) 1100 { 1101 struct cxl_memdev_state *mds = data; 1102 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 1103 struct kernfs_node *state; 1104 1105 /* 1106 * Prevent new irq triggered invocations of the workqueue and 1107 * flush inflight invocations. 1108 */ 1109 mutex_lock(&cxl_mbox->mbox_mutex); 1110 state = mds->security.sanitize_node; 1111 mds->security.sanitize_node = NULL; 1112 mutex_unlock(&cxl_mbox->mbox_mutex); 1113 1114 cancel_delayed_work_sync(&mds->security.poll_dwork); 1115 sysfs_put(state); 1116 } 1117 1118 int devm_cxl_sanitize_setup_notifier(struct device *host, 1119 struct cxl_memdev *cxlmd) 1120 { 1121 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1122 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1123 struct kernfs_node *sec; 1124 1125 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1126 return 0; 1127 1128 /* 1129 * Note, the expectation is that @cxlmd would have failed to be 1130 * created if these sysfs_get_dirent calls fail. 1131 */ 1132 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1133 if (!sec) 1134 return -ENOENT; 1135 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1136 sysfs_put(sec); 1137 if (!mds->security.sanitize_node) 1138 return -ENOENT; 1139 1140 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1141 } 1142 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL"); 1143 1144 __init int cxl_memdev_init(void) 1145 { 1146 dev_t devt; 1147 int rc; 1148 1149 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1150 if (rc) 1151 return rc; 1152 1153 cxl_mem_major = MAJOR(devt); 1154 1155 return 0; 1156 } 1157 1158 void cxl_memdev_exit(void) 1159 { 1160 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1161 } 1162