1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3 #include <linux/libnvdimm.h>
4 #include <linux/unaligned.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/ndctl.h>
8 #include <linux/async.h>
9 #include <linux/slab.h>
10 #include <linux/nd.h>
11 #include "cxlmem.h"
12 #include "cxl.h"
13
14 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
15
16 /**
17 * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology
18 * @host: platform firmware root device
19 * @port: CXL port at the root of a CXL topology
20 *
21 * Return: bridge device that can host cxl_nvdimm objects
22 */
devm_cxl_add_nvdimm_bridge(struct device * host,struct cxl_port * port)23 struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
24 struct cxl_port *port)
25 {
26 return __devm_cxl_add_nvdimm_bridge(host, port);
27 }
28 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL");
29
clear_exclusive(void * mds)30 static void clear_exclusive(void *mds)
31 {
32 clear_exclusive_cxl_commands(mds, exclusive_cmds);
33 }
34
unregister_nvdimm(void * nvdimm)35 static void unregister_nvdimm(void *nvdimm)
36 {
37 nvdimm_delete(nvdimm);
38 }
39
provider_show(struct device * dev,struct device_attribute * attr,char * buf)40 static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
41 {
42 struct nvdimm *nvdimm = to_nvdimm(dev);
43 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
44
45 return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
46 }
47 static DEVICE_ATTR_RO(provider);
48
id_show(struct device * dev,struct device_attribute * attr,char * buf)49 static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
50 {
51 struct nvdimm *nvdimm = to_nvdimm(dev);
52 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
53 struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
54
55 return sysfs_emit(buf, "%lld\n", cxlds->serial);
56 }
57 static DEVICE_ATTR_RO(id);
58
dirty_shutdown_show(struct device * dev,struct device_attribute * attr,char * buf)59 static ssize_t dirty_shutdown_show(struct device *dev,
60 struct device_attribute *attr, char *buf)
61 {
62 struct nvdimm *nvdimm = to_nvdimm(dev);
63 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
64
65 return sysfs_emit(buf, "%llu\n", cxl_nvd->dirty_shutdowns);
66 }
67 static DEVICE_ATTR_RO(dirty_shutdown);
68
69 static struct attribute *cxl_dimm_attributes[] = {
70 &dev_attr_id.attr,
71 &dev_attr_provider.attr,
72 &dev_attr_dirty_shutdown.attr,
73 NULL
74 };
75
76 #define CXL_INVALID_DIRTY_SHUTDOWN_COUNT ULLONG_MAX
cxl_dimm_visible(struct kobject * kobj,struct attribute * a,int n)77 static umode_t cxl_dimm_visible(struct kobject *kobj,
78 struct attribute *a, int n)
79 {
80 if (a == &dev_attr_dirty_shutdown.attr) {
81 struct device *dev = kobj_to_dev(kobj);
82 struct nvdimm *nvdimm = to_nvdimm(dev);
83 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
84
85 if (cxl_nvd->dirty_shutdowns ==
86 CXL_INVALID_DIRTY_SHUTDOWN_COUNT)
87 return 0;
88 }
89
90 return a->mode;
91 }
92
93 static const struct attribute_group cxl_dimm_attribute_group = {
94 .name = "cxl",
95 .attrs = cxl_dimm_attributes,
96 .is_visible = cxl_dimm_visible
97 };
98
99 static const struct attribute_group *cxl_dimm_attribute_groups[] = {
100 &cxl_dimm_attribute_group,
101 NULL
102 };
103
cxl_nvdimm_arm_dirty_shutdown_tracking(struct cxl_nvdimm * cxl_nvd)104 static void cxl_nvdimm_arm_dirty_shutdown_tracking(struct cxl_nvdimm *cxl_nvd)
105 {
106 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
107 struct cxl_dev_state *cxlds = cxlmd->cxlds;
108 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
109 struct device *dev = &cxl_nvd->dev;
110 u32 count;
111
112 /*
113 * Dirty tracking is enabled and exposed to the user, only when:
114 * - dirty shutdown on the device can be set, and,
115 * - the device has a Device GPF DVSEC (albeit unused), and,
116 * - the Get Health Info cmd can retrieve the device's dirty count.
117 */
118 cxl_nvd->dirty_shutdowns = CXL_INVALID_DIRTY_SHUTDOWN_COUNT;
119
120 if (cxl_arm_dirty_shutdown(mds)) {
121 dev_warn(dev, "GPF: could not set dirty shutdown state\n");
122 return;
123 }
124
125 if (!cxl_gpf_get_dvsec(cxlds->dev))
126 return;
127
128 if (cxl_get_dirty_count(mds, &count)) {
129 dev_warn(dev, "GPF: could not retrieve dirty count\n");
130 return;
131 }
132
133 cxl_nvd->dirty_shutdowns = count;
134 }
135
cxl_nvdimm_probe(struct device * dev)136 static int cxl_nvdimm_probe(struct device *dev)
137 {
138 struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
139 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
140 struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
141 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
142 unsigned long flags = 0, cmd_mask = 0;
143 struct nvdimm *nvdimm;
144 int rc;
145
146 if (test_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags))
147 return -EBUSY;
148
149 set_exclusive_cxl_commands(mds, exclusive_cmds);
150 rc = devm_add_action_or_reset(dev, clear_exclusive, mds);
151 if (rc)
152 return rc;
153
154 set_bit(NDD_LABELING, &flags);
155 set_bit(NDD_REGISTER_SYNC, &flags);
156 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
157 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
158 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
159
160 /*
161 * Set dirty shutdown now, with the expectation that the device
162 * clear it upon a successful GPF flow. The exception to this
163 * is upon Viral detection, per CXL 3.2 section 12.4.2.
164 */
165 cxl_nvdimm_arm_dirty_shutdown_tracking(cxl_nvd);
166
167 nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
168 cxl_dimm_attribute_groups, flags,
169 cmd_mask, 0, NULL, cxl_nvd->dev_id,
170 cxl_security_ops, NULL);
171 if (!nvdimm)
172 return -ENOMEM;
173
174 dev_set_drvdata(dev, nvdimm);
175 return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
176 }
177
178 static struct cxl_driver cxl_nvdimm_driver = {
179 .name = "cxl_nvdimm",
180 .probe = cxl_nvdimm_probe,
181 .id = CXL_DEVICE_NVDIMM,
182 .drv = {
183 .suppress_bind_attrs = true,
184 },
185 };
186
cxl_pmem_get_config_size(struct cxl_memdev_state * mds,struct nd_cmd_get_config_size * cmd,unsigned int buf_len)187 static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,
188 struct nd_cmd_get_config_size *cmd,
189 unsigned int buf_len)
190 {
191 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
192
193 if (sizeof(*cmd) > buf_len)
194 return -EINVAL;
195
196 *cmd = (struct nd_cmd_get_config_size){
197 .config_size = mds->lsa_size,
198 .max_xfer =
199 cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa),
200 };
201
202 return 0;
203 }
204
cxl_pmem_get_config_data(struct cxl_memdev_state * mds,struct nd_cmd_get_config_data_hdr * cmd,unsigned int buf_len)205 static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,
206 struct nd_cmd_get_config_data_hdr *cmd,
207 unsigned int buf_len)
208 {
209 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
210 struct cxl_mbox_get_lsa get_lsa;
211 struct cxl_mbox_cmd mbox_cmd;
212 int rc;
213
214 if (sizeof(*cmd) > buf_len)
215 return -EINVAL;
216 if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
217 return -EINVAL;
218
219 get_lsa = (struct cxl_mbox_get_lsa) {
220 .offset = cpu_to_le32(cmd->in_offset),
221 .length = cpu_to_le32(cmd->in_length),
222 };
223 mbox_cmd = (struct cxl_mbox_cmd) {
224 .opcode = CXL_MBOX_OP_GET_LSA,
225 .payload_in = &get_lsa,
226 .size_in = sizeof(get_lsa),
227 .size_out = cmd->in_length,
228 .payload_out = cmd->out_buf,
229 };
230
231 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
232 cmd->status = 0;
233
234 return rc;
235 }
236
cxl_pmem_set_config_data(struct cxl_memdev_state * mds,struct nd_cmd_set_config_hdr * cmd,unsigned int buf_len)237 static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,
238 struct nd_cmd_set_config_hdr *cmd,
239 unsigned int buf_len)
240 {
241 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
242 struct cxl_mbox_set_lsa *set_lsa;
243 struct cxl_mbox_cmd mbox_cmd;
244 int rc;
245
246 if (sizeof(*cmd) > buf_len)
247 return -EINVAL;
248
249 /* 4-byte status follows the input data in the payload */
250 if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
251 return -EINVAL;
252
253 set_lsa =
254 kvzalloc_flex(*set_lsa, data, cmd->in_length);
255 if (!set_lsa)
256 return -ENOMEM;
257
258 *set_lsa = (struct cxl_mbox_set_lsa) {
259 .offset = cpu_to_le32(cmd->in_offset),
260 };
261 memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
262 mbox_cmd = (struct cxl_mbox_cmd) {
263 .opcode = CXL_MBOX_OP_SET_LSA,
264 .payload_in = set_lsa,
265 .size_in = struct_size(set_lsa, data, cmd->in_length),
266 };
267
268 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
269
270 /*
271 * Set "firmware" status (4-packed bytes at the end of the input
272 * payload.
273 */
274 put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
275 kvfree(set_lsa);
276
277 return rc;
278 }
279
cxl_pmem_nvdimm_ctl(struct nvdimm * nvdimm,unsigned int cmd,void * buf,unsigned int buf_len)280 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
281 void *buf, unsigned int buf_len)
282 {
283 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
284 unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
285 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
286 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
287
288 if (!test_bit(cmd, &cmd_mask))
289 return -ENOTTY;
290
291 switch (cmd) {
292 case ND_CMD_GET_CONFIG_SIZE:
293 return cxl_pmem_get_config_size(mds, buf, buf_len);
294 case ND_CMD_GET_CONFIG_DATA:
295 return cxl_pmem_get_config_data(mds, buf, buf_len);
296 case ND_CMD_SET_CONFIG_DATA:
297 return cxl_pmem_set_config_data(mds, buf, buf_len);
298 default:
299 return -ENOTTY;
300 }
301 }
302
cxl_pmem_ctl(struct nvdimm_bus_descriptor * nd_desc,struct nvdimm * nvdimm,unsigned int cmd,void * buf,unsigned int buf_len,int * cmd_rc)303 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
304 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
305 unsigned int buf_len, int *cmd_rc)
306 {
307 /*
308 * No firmware response to translate, let the transport error
309 * code take precedence.
310 */
311 *cmd_rc = 0;
312
313 if (!nvdimm)
314 return -ENOTTY;
315 return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
316 }
317
detach_nvdimm(struct device * dev,void * data)318 static int detach_nvdimm(struct device *dev, void *data)
319 {
320 struct cxl_nvdimm *cxl_nvd;
321 bool release = false;
322
323 if (!is_cxl_nvdimm(dev))
324 return 0;
325
326 scoped_guard(device, dev) {
327 if (dev->driver) {
328 cxl_nvd = to_cxl_nvdimm(dev);
329 if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) {
330 release = true;
331 set_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags);
332 }
333 }
334 }
335 if (release)
336 device_release_driver(dev);
337 return 0;
338 }
339
unregister_nvdimm_bus(void * _cxl_nvb)340 static void unregister_nvdimm_bus(void *_cxl_nvb)
341 {
342 struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
343 struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
344
345 bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
346
347 cxl_nvb->nvdimm_bus = NULL;
348 nvdimm_bus_unregister(nvdimm_bus);
349 }
350
cxl_nvdimm_bridge_probe(struct device * dev)351 static int cxl_nvdimm_bridge_probe(struct device *dev)
352 {
353 struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
354
355 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
356 .provider_name = "CXL",
357 .module = THIS_MODULE,
358 .ndctl = cxl_pmem_ctl,
359 };
360
361 cxl_nvb->nvdimm_bus =
362 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
363
364 if (!cxl_nvb->nvdimm_bus)
365 return -ENOMEM;
366
367 return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
368 }
369
370 static struct cxl_driver cxl_nvdimm_bridge_driver = {
371 .name = "cxl_nvdimm_bridge",
372 .probe = cxl_nvdimm_bridge_probe,
373 .id = CXL_DEVICE_NVDIMM_BRIDGE,
374 .drv = {
375 .probe_type = PROBE_FORCE_SYNCHRONOUS,
376 .suppress_bind_attrs = true,
377 },
378 };
379
unregister_nvdimm_region(void * nd_region)380 static void unregister_nvdimm_region(void *nd_region)
381 {
382 nvdimm_region_delete(nd_region);
383 }
384
cxlr_pmem_remove_resource(void * res)385 static void cxlr_pmem_remove_resource(void *res)
386 {
387 remove_resource(res);
388 }
389
390 struct cxl_pmem_region_info {
391 u64 offset;
392 u64 serial;
393 };
394
cxl_pmem_region_probe(struct device * dev)395 static int cxl_pmem_region_probe(struct device *dev)
396 {
397 struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
398 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
399 struct cxl_region *cxlr = cxlr_pmem->cxlr;
400 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
401 struct cxl_pmem_region_info *info = NULL;
402 struct nd_interleave_set *nd_set;
403 struct nd_region_desc ndr_desc;
404 struct cxl_nvdimm *cxl_nvd;
405 struct nvdimm *nvdimm;
406 struct resource *res;
407 int rc, i = 0;
408
409 memset(&mappings, 0, sizeof(mappings));
410 memset(&ndr_desc, 0, sizeof(ndr_desc));
411
412 res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
413 if (!res)
414 return -ENOMEM;
415
416 res->name = "Persistent Memory";
417 res->start = cxlr_pmem->hpa_range.start;
418 res->end = cxlr_pmem->hpa_range.end;
419 res->flags = IORESOURCE_MEM;
420 res->desc = IORES_DESC_PERSISTENT_MEMORY;
421
422 rc = insert_resource(&iomem_resource, res);
423 if (rc)
424 return rc;
425
426 rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
427 if (rc)
428 return rc;
429
430 ndr_desc.res = res;
431 ndr_desc.provider_data = cxlr_pmem;
432
433 ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
434 ndr_desc.target_node = phys_to_target_node(res->start);
435 if (ndr_desc.target_node == NUMA_NO_NODE) {
436 ndr_desc.target_node = ndr_desc.numa_node;
437 dev_dbg(&cxlr->dev, "changing target node from %d to %d",
438 NUMA_NO_NODE, ndr_desc.target_node);
439 }
440
441 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
442 if (!nd_set)
443 return -ENOMEM;
444
445 ndr_desc.memregion = cxlr->id;
446 set_bit(ND_REGION_CXL, &ndr_desc.flags);
447 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
448
449 info = kmalloc_objs(*info, cxlr_pmem->nr_mappings);
450 if (!info)
451 return -ENOMEM;
452
453 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
454 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
455 struct cxl_memdev *cxlmd = m->cxlmd;
456 struct cxl_dev_state *cxlds = cxlmd->cxlds;
457
458 cxl_nvd = cxlmd->cxl_nvd;
459 nvdimm = dev_get_drvdata(&cxl_nvd->dev);
460 if (!nvdimm) {
461 dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
462 dev_name(&cxlmd->dev));
463 rc = -ENODEV;
464 goto out_nvd;
465 }
466
467 if (cxlds->serial == 0) {
468 /* include missing alongside invalid in this error message. */
469 dev_err(dev, "%s: invalid or missing serial number\n",
470 dev_name(&cxlmd->dev));
471 rc = -ENXIO;
472 goto out_nvd;
473 }
474 info[i].serial = cxlds->serial;
475 info[i].offset = m->start;
476
477 m->cxl_nvd = cxl_nvd;
478 mappings[i] = (struct nd_mapping_desc) {
479 .nvdimm = nvdimm,
480 .start = m->start,
481 .size = m->size,
482 .position = i,
483 };
484 }
485 ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
486 ndr_desc.mapping = mappings;
487
488 /*
489 * TODO enable CXL labels which skip the need for 'interleave-set cookie'
490 */
491 nd_set->cookie1 =
492 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
493 nd_set->cookie2 = nd_set->cookie1;
494 ndr_desc.nd_set = nd_set;
495
496 cxlr_pmem->nd_region =
497 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
498 if (!cxlr_pmem->nd_region) {
499 rc = -ENOMEM;
500 goto out_nvd;
501 }
502
503 rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
504 cxlr_pmem->nd_region);
505 out_nvd:
506 kfree(info);
507
508 return rc;
509 }
510
511 static struct cxl_driver cxl_pmem_region_driver = {
512 .name = "cxl_pmem_region",
513 .probe = cxl_pmem_region_probe,
514 .id = CXL_DEVICE_PMEM_REGION,
515 .drv = {
516 .suppress_bind_attrs = true,
517 },
518 };
519
cxl_pmem_init(void)520 static __init int cxl_pmem_init(void)
521 {
522 int rc;
523
524 set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
525 set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
526
527 rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
528 if (rc)
529 return rc;
530
531 rc = cxl_driver_register(&cxl_nvdimm_driver);
532 if (rc)
533 goto err_nvdimm;
534
535 rc = cxl_driver_register(&cxl_pmem_region_driver);
536 if (rc)
537 goto err_region;
538
539 return 0;
540
541 err_region:
542 cxl_driver_unregister(&cxl_nvdimm_driver);
543 err_nvdimm:
544 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
545 return rc;
546 }
547
cxl_pmem_exit(void)548 static __exit void cxl_pmem_exit(void)
549 {
550 cxl_driver_unregister(&cxl_pmem_region_driver);
551 cxl_driver_unregister(&cxl_nvdimm_driver);
552 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
553 }
554
555 MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support");
556 MODULE_LICENSE("GPL v2");
557 subsys_initcall(cxl_pmem_init);
558 module_exit(cxl_pmem_exit);
559 MODULE_IMPORT_NS("CXL");
560 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
561 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
562 MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);
563