1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/security.h>
4 #include <linux/debugfs.h>
5 #include <linux/ktime.h>
6 #include <linux/mutex.h>
7 #include <linux/unaligned.h>
8 #include <cxlpci.h>
9 #include <cxlmem.h>
10 #include <cxl.h>
11
12 #include "core.h"
13 #include "trace.h"
14 #include "mce.h"
15
16 static bool cxl_raw_allow_all;
17
18 /**
19 * DOC: cxl mbox
20 *
21 * Core implementation of the CXL 2.0 Type-3 Memory Device Mailbox. The
22 * implementation is used by the cxl_pci driver to initialize the device
23 * and implement the cxl_mem.h IOCTL UAPI. It also implements the
24 * backend of the cxl_pmem_ctl() transport for LIBNVDIMM.
25 */
26
27 #define cxl_for_each_cmd(cmd) \
28 for ((cmd) = &cxl_mem_commands[0]; \
29 ((cmd) - cxl_mem_commands) < ARRAY_SIZE(cxl_mem_commands); (cmd)++)
30
31 #define CXL_CMD(_id, sin, sout, _flags) \
32 [CXL_MEM_COMMAND_ID_##_id] = { \
33 .info = { \
34 .id = CXL_MEM_COMMAND_ID_##_id, \
35 .size_in = sin, \
36 .size_out = sout, \
37 }, \
38 .opcode = CXL_MBOX_OP_##_id, \
39 .flags = _flags, \
40 }
41
42 #define CXL_VARIABLE_PAYLOAD ~0U
43 /*
44 * This table defines the supported mailbox commands for the driver. This table
45 * is made up of a UAPI structure. Non-negative values as parameters in the
46 * table will be validated against the user's input. For example, if size_in is
47 * 0, and the user passed in 1, it is an error.
48 */
49 static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
50 CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
51 #ifdef CONFIG_CXL_MEM_RAW_COMMANDS
52 CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),
53 #endif
54 CXL_CMD(GET_SUPPORTED_LOGS, 0, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
55 CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
56 CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
57 CXL_CMD(GET_LSA, 0x8, CXL_VARIABLE_PAYLOAD, 0),
58 CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
59 CXL_CMD(GET_LOG, 0x18, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
60 CXL_CMD(GET_LOG_CAPS, 0x10, 0x4, 0),
61 CXL_CMD(CLEAR_LOG, 0x10, 0, 0),
62 CXL_CMD(GET_SUP_LOG_SUBLIST, 0x2, CXL_VARIABLE_PAYLOAD, 0),
63 CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
64 CXL_CMD(SET_LSA, CXL_VARIABLE_PAYLOAD, 0, 0),
65 CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
66 CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
67 CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
68 CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
69 CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
70 CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),
71 };
72
73 /*
74 * Commands that RAW doesn't permit. The rationale for each:
75 *
76 * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /
77 * coordination of transaction timeout values at the root bridge level.
78 *
79 * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live
80 * and needs to be coordinated with HDM updates.
81 *
82 * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the
83 * driver and any writes from userspace invalidates those contents.
84 *
85 * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes
86 * to the device after it is marked clean, userspace can not make that
87 * assertion.
88 *
89 * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that
90 * is kept up to date with patrol notifications and error management.
91 *
92 * CXL_MBOX_OP_[GET_,INJECT_,CLEAR_]POISON: These commands require kernel
93 * driver orchestration for safety.
94 */
95 static u16 cxl_disabled_raw_commands[] = {
96 CXL_MBOX_OP_ACTIVATE_FW,
97 CXL_MBOX_OP_SET_PARTITION_INFO,
98 CXL_MBOX_OP_SET_LSA,
99 CXL_MBOX_OP_SET_SHUTDOWN_STATE,
100 CXL_MBOX_OP_SCAN_MEDIA,
101 CXL_MBOX_OP_GET_SCAN_MEDIA,
102 CXL_MBOX_OP_GET_POISON,
103 CXL_MBOX_OP_INJECT_POISON,
104 CXL_MBOX_OP_CLEAR_POISON,
105 };
106
107 /*
108 * Command sets that RAW doesn't permit. All opcodes in this set are
109 * disabled because they pass plain text security payloads over the
110 * user/kernel boundary. This functionality is intended to be wrapped
111 * behind the keys ABI which allows for encrypted payloads in the UAPI
112 */
113 static u8 security_command_sets[] = {
114 0x44, /* Sanitize */
115 0x45, /* Persistent Memory Data-at-rest Security */
116 0x46, /* Security Passthrough */
117 };
118
cxl_is_security_command(u16 opcode)119 static bool cxl_is_security_command(u16 opcode)
120 {
121 int i;
122
123 for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)
124 if (security_command_sets[i] == (opcode >> 8))
125 return true;
126 return false;
127 }
128
cxl_set_security_cmd_enabled(struct cxl_security_state * security,u16 opcode)129 static void cxl_set_security_cmd_enabled(struct cxl_security_state *security,
130 u16 opcode)
131 {
132 switch (opcode) {
133 case CXL_MBOX_OP_SANITIZE:
134 set_bit(CXL_SEC_ENABLED_SANITIZE, security->enabled_cmds);
135 break;
136 case CXL_MBOX_OP_SECURE_ERASE:
137 set_bit(CXL_SEC_ENABLED_SECURE_ERASE,
138 security->enabled_cmds);
139 break;
140 case CXL_MBOX_OP_GET_SECURITY_STATE:
141 set_bit(CXL_SEC_ENABLED_GET_SECURITY_STATE,
142 security->enabled_cmds);
143 break;
144 case CXL_MBOX_OP_SET_PASSPHRASE:
145 set_bit(CXL_SEC_ENABLED_SET_PASSPHRASE,
146 security->enabled_cmds);
147 break;
148 case CXL_MBOX_OP_DISABLE_PASSPHRASE:
149 set_bit(CXL_SEC_ENABLED_DISABLE_PASSPHRASE,
150 security->enabled_cmds);
151 break;
152 case CXL_MBOX_OP_UNLOCK:
153 set_bit(CXL_SEC_ENABLED_UNLOCK, security->enabled_cmds);
154 break;
155 case CXL_MBOX_OP_FREEZE_SECURITY:
156 set_bit(CXL_SEC_ENABLED_FREEZE_SECURITY,
157 security->enabled_cmds);
158 break;
159 case CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE:
160 set_bit(CXL_SEC_ENABLED_PASSPHRASE_SECURE_ERASE,
161 security->enabled_cmds);
162 break;
163 default:
164 break;
165 }
166 }
167
cxl_is_poison_command(u16 opcode)168 static bool cxl_is_poison_command(u16 opcode)
169 {
170 #define CXL_MBOX_OP_POISON_CMDS 0x43
171
172 if ((opcode >> 8) == CXL_MBOX_OP_POISON_CMDS)
173 return true;
174
175 return false;
176 }
177
cxl_set_poison_cmd_enabled(struct cxl_poison_state * poison,u16 opcode)178 static void cxl_set_poison_cmd_enabled(struct cxl_poison_state *poison,
179 u16 opcode)
180 {
181 switch (opcode) {
182 case CXL_MBOX_OP_GET_POISON:
183 set_bit(CXL_POISON_ENABLED_LIST, poison->enabled_cmds);
184 break;
185 case CXL_MBOX_OP_INJECT_POISON:
186 set_bit(CXL_POISON_ENABLED_INJECT, poison->enabled_cmds);
187 break;
188 case CXL_MBOX_OP_CLEAR_POISON:
189 set_bit(CXL_POISON_ENABLED_CLEAR, poison->enabled_cmds);
190 break;
191 case CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS:
192 set_bit(CXL_POISON_ENABLED_SCAN_CAPS, poison->enabled_cmds);
193 break;
194 case CXL_MBOX_OP_SCAN_MEDIA:
195 set_bit(CXL_POISON_ENABLED_SCAN_MEDIA, poison->enabled_cmds);
196 break;
197 case CXL_MBOX_OP_GET_SCAN_MEDIA:
198 set_bit(CXL_POISON_ENABLED_SCAN_RESULTS, poison->enabled_cmds);
199 break;
200 default:
201 break;
202 }
203 }
204
cxl_mem_find_command(u16 opcode)205 static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
206 {
207 struct cxl_mem_command *c;
208
209 cxl_for_each_cmd(c)
210 if (c->opcode == opcode)
211 return c;
212
213 return NULL;
214 }
215
cxl_mem_opcode_to_name(u16 opcode)216 static const char *cxl_mem_opcode_to_name(u16 opcode)
217 {
218 struct cxl_mem_command *c;
219
220 c = cxl_mem_find_command(opcode);
221 if (!c)
222 return NULL;
223
224 return cxl_command_names[c->info.id].name;
225 }
226
227 /**
228 * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command
229 * @cxl_mbox: CXL mailbox context
230 * @mbox_cmd: initialized command to execute
231 *
232 * Context: Any context.
233 * Return:
234 * * %>=0 - Number of bytes returned in @out.
235 * * %-E2BIG - Payload is too large for hardware.
236 * * %-EBUSY - Couldn't acquire exclusive mailbox access.
237 * * %-EFAULT - Hardware error occurred.
238 * * %-ENXIO - Command completed, but device reported an error.
239 * * %-EIO - Unexpected output size.
240 *
241 * Mailbox commands may execute successfully yet the device itself reported an
242 * error. While this distinction can be useful for commands from userspace, the
243 * kernel will only be able to use results when both are successful.
244 */
cxl_internal_send_cmd(struct cxl_mailbox * cxl_mbox,struct cxl_mbox_cmd * mbox_cmd)245 int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
246 struct cxl_mbox_cmd *mbox_cmd)
247 {
248 size_t out_size, min_out;
249 int rc;
250
251 if (mbox_cmd->size_in > cxl_mbox->payload_size ||
252 mbox_cmd->size_out > cxl_mbox->payload_size)
253 return -E2BIG;
254
255 out_size = mbox_cmd->size_out;
256 min_out = mbox_cmd->min_out;
257 rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
258 /*
259 * EIO is reserved for a payload size mismatch and mbox_send()
260 * may not return this error.
261 */
262 if (WARN_ONCE(rc == -EIO, "Bad return code: -EIO"))
263 return -ENXIO;
264 if (rc)
265 return rc;
266
267 if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS &&
268 mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND)
269 return cxl_mbox_cmd_rc2errno(mbox_cmd);
270
271 if (!out_size)
272 return 0;
273
274 /*
275 * Variable sized output needs to at least satisfy the caller's
276 * minimum if not the fully requested size.
277 */
278 if (min_out == 0)
279 min_out = out_size;
280
281 if (mbox_cmd->size_out < min_out)
282 return -EIO;
283 return 0;
284 }
285 EXPORT_SYMBOL_NS_GPL(cxl_internal_send_cmd, "CXL");
286
cxl_mem_raw_command_allowed(u16 opcode)287 static bool cxl_mem_raw_command_allowed(u16 opcode)
288 {
289 int i;
290
291 if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))
292 return false;
293
294 if (security_locked_down(LOCKDOWN_PCI_ACCESS))
295 return false;
296
297 if (cxl_raw_allow_all)
298 return true;
299
300 if (cxl_is_security_command(opcode))
301 return false;
302
303 for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)
304 if (cxl_disabled_raw_commands[i] == opcode)
305 return false;
306
307 return true;
308 }
309
310 /**
311 * cxl_payload_from_user_allowed() - Check contents of in_payload.
312 * @opcode: The mailbox command opcode.
313 * @payload_in: Pointer to the input payload passed in from user space.
314 * @in_size: Size of @payload_in in bytes.
315 *
316 * Return:
317 * * true - payload_in passes check for @opcode.
318 * * false - payload_in contains invalid or unsupported values.
319 *
320 * The driver may inspect payload contents before sending a mailbox
321 * command from user space to the device. The intent is to reject
322 * commands with input payloads that are known to be unsafe. This
323 * check is not intended to replace the users careful selection of
324 * mailbox command parameters and makes no guarantee that the user
325 * command will succeed, nor that it is appropriate.
326 *
327 * The specific checks are determined by the opcode.
328 */
cxl_payload_from_user_allowed(u16 opcode,void * payload_in,size_t in_size)329 static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in,
330 size_t in_size)
331 {
332 switch (opcode) {
333 case CXL_MBOX_OP_SET_PARTITION_INFO: {
334 struct cxl_mbox_set_partition_info *pi = payload_in;
335
336 if (in_size < sizeof(*pi))
337 return false;
338 if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG)
339 return false;
340 break;
341 }
342 case CXL_MBOX_OP_CLEAR_LOG: {
343 const uuid_t *uuid = (uuid_t *)payload_in;
344
345 if (in_size < sizeof(uuid_t))
346 return false;
347 /*
348 * Restrict the ‘Clear log’ action to only apply to
349 * Vendor debug logs.
350 */
351 return uuid_equal(uuid, &DEFINE_CXL_VENDOR_DEBUG_UUID);
352 }
353 default:
354 break;
355 }
356 return true;
357 }
358
cxl_mbox_cmd_ctor(struct cxl_mbox_cmd * mbox_cmd,struct cxl_mailbox * cxl_mbox,u16 opcode,size_t in_size,size_t out_size,u64 in_payload)359 static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd,
360 struct cxl_mailbox *cxl_mbox, u16 opcode,
361 size_t in_size, size_t out_size, u64 in_payload)
362 {
363 *mbox_cmd = (struct cxl_mbox_cmd) {
364 .opcode = opcode,
365 .size_in = in_size,
366 };
367
368 if (in_size) {
369 mbox_cmd->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
370 in_size);
371 if (IS_ERR(mbox_cmd->payload_in))
372 return PTR_ERR(mbox_cmd->payload_in);
373
374 if (!cxl_payload_from_user_allowed(opcode, mbox_cmd->payload_in,
375 in_size)) {
376 dev_dbg(cxl_mbox->host, "%s: input payload not allowed\n",
377 cxl_mem_opcode_to_name(opcode));
378 kvfree(mbox_cmd->payload_in);
379 return -EBUSY;
380 }
381 }
382
383 /* Prepare to handle a full payload for variable sized output */
384 if (out_size == CXL_VARIABLE_PAYLOAD)
385 mbox_cmd->size_out = cxl_mbox->payload_size;
386 else
387 mbox_cmd->size_out = out_size;
388
389 if (mbox_cmd->size_out) {
390 mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL);
391 if (!mbox_cmd->payload_out) {
392 kvfree(mbox_cmd->payload_in);
393 return -ENOMEM;
394 }
395 }
396 return 0;
397 }
398
cxl_mbox_cmd_dtor(struct cxl_mbox_cmd * mbox)399 static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
400 {
401 kvfree(mbox->payload_in);
402 kvfree(mbox->payload_out);
403 }
404
cxl_to_mem_cmd_raw(struct cxl_mem_command * mem_cmd,const struct cxl_send_command * send_cmd,struct cxl_mailbox * cxl_mbox)405 static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
406 const struct cxl_send_command *send_cmd,
407 struct cxl_mailbox *cxl_mbox)
408 {
409 if (send_cmd->raw.rsvd)
410 return -EINVAL;
411
412 /*
413 * Unlike supported commands, the output size of RAW commands
414 * gets passed along without further checking, so it must be
415 * validated here.
416 */
417 if (send_cmd->out.size > cxl_mbox->payload_size)
418 return -EINVAL;
419
420 if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
421 return -EPERM;
422
423 dev_WARN_ONCE(cxl_mbox->host, true, "raw command path used\n");
424
425 *mem_cmd = (struct cxl_mem_command) {
426 .info = {
427 .id = CXL_MEM_COMMAND_ID_RAW,
428 .size_in = send_cmd->in.size,
429 .size_out = send_cmd->out.size,
430 },
431 .opcode = send_cmd->raw.opcode
432 };
433
434 return 0;
435 }
436
cxl_to_mem_cmd(struct cxl_mem_command * mem_cmd,const struct cxl_send_command * send_cmd,struct cxl_mailbox * cxl_mbox)437 static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
438 const struct cxl_send_command *send_cmd,
439 struct cxl_mailbox *cxl_mbox)
440 {
441 struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
442 const struct cxl_command_info *info = &c->info;
443
444 if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
445 return -EINVAL;
446
447 if (send_cmd->rsvd)
448 return -EINVAL;
449
450 if (send_cmd->in.rsvd || send_cmd->out.rsvd)
451 return -EINVAL;
452
453 /* Check that the command is enabled for hardware */
454 if (!test_bit(info->id, cxl_mbox->enabled_cmds))
455 return -ENOTTY;
456
457 /* Check that the command is not claimed for exclusive kernel use */
458 if (test_bit(info->id, cxl_mbox->exclusive_cmds))
459 return -EBUSY;
460
461 /* Check the input buffer is the expected size */
462 if ((info->size_in != CXL_VARIABLE_PAYLOAD) &&
463 (info->size_in != send_cmd->in.size))
464 return -ENOMEM;
465
466 /* Check the output buffer is at least large enough */
467 if ((info->size_out != CXL_VARIABLE_PAYLOAD) &&
468 (send_cmd->out.size < info->size_out))
469 return -ENOMEM;
470
471 *mem_cmd = (struct cxl_mem_command) {
472 .info = {
473 .id = info->id,
474 .flags = info->flags,
475 .size_in = send_cmd->in.size,
476 .size_out = send_cmd->out.size,
477 },
478 .opcode = c->opcode
479 };
480
481 return 0;
482 }
483
484 /**
485 * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
486 * @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
487 * @cxl_mbox: CXL mailbox context
488 * @send_cmd: &struct cxl_send_command copied in from userspace.
489 *
490 * Return:
491 * * %0 - @out_cmd is ready to send.
492 * * %-ENOTTY - Invalid command specified.
493 * * %-EINVAL - Reserved fields or invalid values were used.
494 * * %-ENOMEM - Input or output buffer wasn't sized properly.
495 * * %-EPERM - Attempted to use a protected command.
496 * * %-EBUSY - Kernel has claimed exclusive access to this opcode
497 *
498 * The result of this command is a fully validated command in @mbox_cmd that is
499 * safe to send to the hardware.
500 */
cxl_validate_cmd_from_user(struct cxl_mbox_cmd * mbox_cmd,struct cxl_mailbox * cxl_mbox,const struct cxl_send_command * send_cmd)501 static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
502 struct cxl_mailbox *cxl_mbox,
503 const struct cxl_send_command *send_cmd)
504 {
505 struct cxl_mem_command mem_cmd;
506 int rc;
507
508 if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
509 return -ENOTTY;
510
511 /*
512 * The user can never specify an input payload larger than what hardware
513 * supports, but output can be arbitrarily large (simply write out as
514 * much data as the hardware provides).
515 */
516 if (send_cmd->in.size > cxl_mbox->payload_size)
517 return -EINVAL;
518
519 /* Sanitize and construct a cxl_mem_command */
520 if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
521 rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, cxl_mbox);
522 else
523 rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, cxl_mbox);
524
525 if (rc)
526 return rc;
527
528 /* Sanitize and construct a cxl_mbox_cmd */
529 return cxl_mbox_cmd_ctor(mbox_cmd, cxl_mbox, mem_cmd.opcode,
530 mem_cmd.info.size_in, mem_cmd.info.size_out,
531 send_cmd->in.payload);
532 }
533
cxl_query_cmd(struct cxl_mailbox * cxl_mbox,struct cxl_mem_query_commands __user * q)534 int cxl_query_cmd(struct cxl_mailbox *cxl_mbox,
535 struct cxl_mem_query_commands __user *q)
536 {
537 struct device *dev = cxl_mbox->host;
538 struct cxl_mem_command *cmd;
539 u32 n_commands;
540 int j = 0;
541
542 dev_dbg(dev, "Query IOCTL\n");
543
544 if (get_user(n_commands, &q->n_commands))
545 return -EFAULT;
546
547 /* returns the total number if 0 elements are requested. */
548 if (n_commands == 0)
549 return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
550
551 /*
552 * otherwise, return min(n_commands, total commands) cxl_command_info
553 * structures.
554 */
555 cxl_for_each_cmd(cmd) {
556 struct cxl_command_info info = cmd->info;
557
558 if (test_bit(info.id, cxl_mbox->enabled_cmds))
559 info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
560 if (test_bit(info.id, cxl_mbox->exclusive_cmds))
561 info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
562
563 if (copy_to_user(&q->commands[j++], &info, sizeof(info)))
564 return -EFAULT;
565
566 if (j == n_commands)
567 break;
568 }
569
570 return 0;
571 }
572
573 /**
574 * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
575 * @cxl_mbox: The mailbox context for the operation.
576 * @mbox_cmd: The validated mailbox command.
577 * @out_payload: Pointer to userspace's output payload.
578 * @size_out: (Input) Max payload size to copy out.
579 * (Output) Payload size hardware generated.
580 * @retval: Hardware generated return code from the operation.
581 *
582 * Return:
583 * * %0 - Mailbox transaction succeeded. This implies the mailbox
584 * protocol completed successfully not that the operation itself
585 * was successful.
586 * * %-ENOMEM - Couldn't allocate a bounce buffer.
587 * * %-EFAULT - Something happened with copy_to/from_user.
588 * * %-EINTR - Mailbox acquisition interrupted.
589 * * %-EXXX - Transaction level failures.
590 *
591 * Dispatches a mailbox command on behalf of a userspace request.
592 * The output payload is copied to userspace.
593 *
594 * See cxl_send_cmd().
595 */
handle_mailbox_cmd_from_user(struct cxl_mailbox * cxl_mbox,struct cxl_mbox_cmd * mbox_cmd,u64 out_payload,s32 * size_out,u32 * retval)596 static int handle_mailbox_cmd_from_user(struct cxl_mailbox *cxl_mbox,
597 struct cxl_mbox_cmd *mbox_cmd,
598 u64 out_payload, s32 *size_out,
599 u32 *retval)
600 {
601 struct device *dev = cxl_mbox->host;
602 int rc;
603
604 dev_dbg(dev,
605 "Submitting %s command for user\n"
606 "\topcode: %x\n"
607 "\tsize: %zx\n",
608 cxl_mem_opcode_to_name(mbox_cmd->opcode),
609 mbox_cmd->opcode, mbox_cmd->size_in);
610
611 rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
612 if (rc)
613 goto out;
614
615 /*
616 * @size_out contains the max size that's allowed to be written back out
617 * to userspace. While the payload may have written more output than
618 * this it will have to be ignored.
619 */
620 if (mbox_cmd->size_out) {
621 dev_WARN_ONCE(dev, mbox_cmd->size_out > *size_out,
622 "Invalid return size\n");
623 if (copy_to_user(u64_to_user_ptr(out_payload),
624 mbox_cmd->payload_out, mbox_cmd->size_out)) {
625 rc = -EFAULT;
626 goto out;
627 }
628 }
629
630 *size_out = mbox_cmd->size_out;
631 *retval = mbox_cmd->return_code;
632
633 out:
634 cxl_mbox_cmd_dtor(mbox_cmd);
635 return rc;
636 }
637
cxl_send_cmd(struct cxl_mailbox * cxl_mbox,struct cxl_send_command __user * s)638 int cxl_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_send_command __user *s)
639 {
640 struct device *dev = cxl_mbox->host;
641 struct cxl_send_command send;
642 struct cxl_mbox_cmd mbox_cmd;
643 int rc;
644
645 dev_dbg(dev, "Send IOCTL\n");
646
647 if (copy_from_user(&send, s, sizeof(send)))
648 return -EFAULT;
649
650 rc = cxl_validate_cmd_from_user(&mbox_cmd, cxl_mbox, &send);
651 if (rc)
652 return rc;
653
654 rc = handle_mailbox_cmd_from_user(cxl_mbox, &mbox_cmd, send.out.payload,
655 &send.out.size, &send.retval);
656 if (rc)
657 return rc;
658
659 if (copy_to_user(s, &send, sizeof(send)))
660 return -EFAULT;
661
662 return 0;
663 }
664
cxl_xfer_log(struct cxl_memdev_state * mds,uuid_t * uuid,u32 * size,u8 * out)665 static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
666 u32 *size, u8 *out)
667 {
668 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
669 u32 remaining = *size;
670 u32 offset = 0;
671
672 while (remaining) {
673 u32 xfer_size = min_t(u32, remaining, cxl_mbox->payload_size);
674 struct cxl_mbox_cmd mbox_cmd;
675 struct cxl_mbox_get_log log;
676 int rc;
677
678 log = (struct cxl_mbox_get_log) {
679 .uuid = *uuid,
680 .offset = cpu_to_le32(offset),
681 .length = cpu_to_le32(xfer_size),
682 };
683
684 mbox_cmd = (struct cxl_mbox_cmd) {
685 .opcode = CXL_MBOX_OP_GET_LOG,
686 .size_in = sizeof(log),
687 .payload_in = &log,
688 .size_out = xfer_size,
689 .payload_out = out,
690 };
691
692 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
693
694 /*
695 * The output payload length that indicates the number
696 * of valid bytes can be smaller than the Log buffer
697 * size.
698 */
699 if (rc == -EIO && mbox_cmd.size_out < xfer_size) {
700 offset += mbox_cmd.size_out;
701 break;
702 }
703
704 if (rc < 0)
705 return rc;
706
707 out += xfer_size;
708 remaining -= xfer_size;
709 offset += xfer_size;
710 }
711
712 *size = offset;
713
714 return 0;
715 }
716
check_features_opcodes(u16 opcode,int * ro_cmds,int * wr_cmds)717 static int check_features_opcodes(u16 opcode, int *ro_cmds, int *wr_cmds)
718 {
719 switch (opcode) {
720 case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
721 case CXL_MBOX_OP_GET_FEATURE:
722 (*ro_cmds)++;
723 return 1;
724 case CXL_MBOX_OP_SET_FEATURE:
725 (*wr_cmds)++;
726 return 1;
727 default:
728 return 0;
729 }
730 }
731
732 /* 'Get Supported Features' and 'Get Feature' */
733 #define MAX_FEATURES_READ_CMDS 2
set_features_cap(struct cxl_mailbox * cxl_mbox,int ro_cmds,int wr_cmds)734 static void set_features_cap(struct cxl_mailbox *cxl_mbox,
735 int ro_cmds, int wr_cmds)
736 {
737 /* Setting up Features capability while walking the CEL */
738 if (ro_cmds == MAX_FEATURES_READ_CMDS) {
739 if (wr_cmds)
740 cxl_mbox->feat_cap = CXL_FEATURES_RW;
741 else
742 cxl_mbox->feat_cap = CXL_FEATURES_RO;
743 }
744 }
745
746 /**
747 * cxl_walk_cel() - Walk through the Command Effects Log.
748 * @mds: The driver data for the operation
749 * @size: Length of the Command Effects Log.
750 * @cel: CEL
751 *
752 * Iterate over each entry in the CEL and determine if the driver supports the
753 * command. If so, the command is enabled for the device and can be used later.
754 */
cxl_walk_cel(struct cxl_memdev_state * mds,size_t size,u8 * cel)755 static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
756 {
757 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
758 struct cxl_cel_entry *cel_entry;
759 const int cel_entries = size / sizeof(*cel_entry);
760 struct device *dev = mds->cxlds.dev;
761 int i, ro_cmds = 0, wr_cmds = 0;
762
763 cel_entry = (struct cxl_cel_entry *) cel;
764
765 for (i = 0; i < cel_entries; i++) {
766 u16 opcode = le16_to_cpu(cel_entry[i].opcode);
767 struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
768 int enabled = 0;
769
770 if (cmd) {
771 set_bit(cmd->info.id, cxl_mbox->enabled_cmds);
772 enabled++;
773 }
774
775 enabled += check_features_opcodes(opcode, &ro_cmds,
776 &wr_cmds);
777
778 if (cxl_is_poison_command(opcode)) {
779 cxl_set_poison_cmd_enabled(&mds->poison, opcode);
780 enabled++;
781 }
782
783 if (cxl_is_security_command(opcode)) {
784 cxl_set_security_cmd_enabled(&mds->security, opcode);
785 enabled++;
786 }
787
788 dev_dbg(dev, "Opcode 0x%04x %s\n", opcode,
789 enabled ? "enabled" : "unsupported by driver");
790 }
791
792 set_features_cap(cxl_mbox, ro_cmds, wr_cmds);
793 }
794
cxl_get_gsl(struct cxl_memdev_state * mds)795 static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
796 {
797 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
798 struct cxl_mbox_get_supported_logs *ret;
799 struct cxl_mbox_cmd mbox_cmd;
800 int rc;
801
802 ret = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
803 if (!ret)
804 return ERR_PTR(-ENOMEM);
805
806 mbox_cmd = (struct cxl_mbox_cmd) {
807 .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
808 .size_out = cxl_mbox->payload_size,
809 .payload_out = ret,
810 /* At least the record number field must be valid */
811 .min_out = 2,
812 };
813 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
814 if (rc < 0) {
815 kvfree(ret);
816 return ERR_PTR(rc);
817 }
818
819
820 return ret;
821 }
822
823 enum {
824 CEL_UUID,
825 VENDOR_DEBUG_UUID,
826 };
827
828 /* See CXL 2.0 Table 170. Get Log Input Payload */
829 static const uuid_t log_uuid[] = {
830 [CEL_UUID] = DEFINE_CXL_CEL_UUID,
831 [VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,
832 };
833
834 /**
835 * cxl_enumerate_cmds() - Enumerate commands for a device.
836 * @mds: The driver data for the operation
837 *
838 * Returns 0 if enumerate completed successfully.
839 *
840 * CXL devices have optional support for certain commands. This function will
841 * determine the set of supported commands for the hardware and update the
842 * enabled_cmds bitmap in the @mds.
843 */
cxl_enumerate_cmds(struct cxl_memdev_state * mds)844 int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
845 {
846 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
847 struct cxl_mbox_get_supported_logs *gsl;
848 struct device *dev = mds->cxlds.dev;
849 struct cxl_mem_command *cmd;
850 int i, rc;
851
852 gsl = cxl_get_gsl(mds);
853 if (IS_ERR(gsl))
854 return PTR_ERR(gsl);
855
856 rc = -ENOENT;
857 for (i = 0; i < le16_to_cpu(gsl->entries); i++) {
858 u32 size = le32_to_cpu(gsl->entry[i].size);
859 uuid_t uuid = gsl->entry[i].uuid;
860 u8 *log;
861
862 dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);
863
864 if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))
865 continue;
866
867 log = kvmalloc(size, GFP_KERNEL);
868 if (!log) {
869 rc = -ENOMEM;
870 goto out;
871 }
872
873 rc = cxl_xfer_log(mds, &uuid, &size, log);
874 if (rc) {
875 kvfree(log);
876 goto out;
877 }
878
879 cxl_walk_cel(mds, size, log);
880 kvfree(log);
881
882 /* In case CEL was bogus, enable some default commands. */
883 cxl_for_each_cmd(cmd)
884 if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
885 set_bit(cmd->info.id, cxl_mbox->enabled_cmds);
886
887 /* Found the required CEL */
888 rc = 0;
889 }
890 out:
891 kvfree(gsl);
892 return rc;
893 }
894 EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, "CXL");
895
cxl_event_trace_record(const struct cxl_memdev * cxlmd,enum cxl_event_log_type type,enum cxl_event_type event_type,const uuid_t * uuid,union cxl_event * evt)896 void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
897 enum cxl_event_log_type type,
898 enum cxl_event_type event_type,
899 const uuid_t *uuid, union cxl_event *evt)
900 {
901 if (event_type == CXL_CPER_EVENT_MEM_MODULE) {
902 trace_cxl_memory_module(cxlmd, type, &evt->mem_module);
903 return;
904 }
905 if (event_type == CXL_CPER_EVENT_GENERIC) {
906 trace_cxl_generic_event(cxlmd, type, uuid, &evt->generic);
907 return;
908 }
909 if (event_type == CXL_CPER_EVENT_MEM_SPARING) {
910 trace_cxl_memory_sparing(cxlmd, type, &evt->mem_sparing);
911 return;
912 }
913
914 if (trace_cxl_general_media_enabled() || trace_cxl_dram_enabled()) {
915 u64 dpa, hpa = ULLONG_MAX, hpa_alias = ULLONG_MAX;
916 struct cxl_region *cxlr;
917
918 /*
919 * These trace points are annotated with HPA and region
920 * translations. Take topology mutation locks and lookup
921 * { HPA, REGION } from { DPA, MEMDEV } in the event record.
922 */
923 guard(rwsem_read)(&cxl_rwsem.region);
924 guard(rwsem_read)(&cxl_rwsem.dpa);
925
926 dpa = le64_to_cpu(evt->media_hdr.phys_addr) & CXL_DPA_MASK;
927 cxlr = cxl_dpa_to_region(cxlmd, dpa);
928 if (cxlr) {
929 u64 cache_size = cxlr->params.cache_size;
930
931 hpa = cxl_dpa_to_hpa(cxlr, cxlmd, dpa);
932 if (cache_size)
933 hpa_alias = hpa - cache_size;
934 }
935
936 if (event_type == CXL_CPER_EVENT_GEN_MEDIA) {
937 if (cxl_store_rec_gen_media((struct cxl_memdev *)cxlmd, evt))
938 dev_dbg(&cxlmd->dev, "CXL store rec_gen_media failed\n");
939
940 if (evt->gen_media.media_hdr.descriptor &
941 CXL_GMER_EVT_DESC_THRESHOLD_EVENT)
942 WARN_ON_ONCE((evt->gen_media.media_hdr.type &
943 CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) &&
944 !get_unaligned_le24(evt->gen_media.cme_count));
945 else
946 WARN_ON_ONCE(evt->gen_media.media_hdr.type &
947 CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE);
948
949 trace_cxl_general_media(cxlmd, type, cxlr, hpa,
950 hpa_alias, &evt->gen_media);
951 } else if (event_type == CXL_CPER_EVENT_DRAM) {
952 if (cxl_store_rec_dram((struct cxl_memdev *)cxlmd, evt))
953 dev_dbg(&cxlmd->dev, "CXL store rec_dram failed\n");
954
955 if (evt->dram.media_hdr.descriptor &
956 CXL_GMER_EVT_DESC_THRESHOLD_EVENT)
957 WARN_ON_ONCE((evt->dram.media_hdr.type &
958 CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) &&
959 !get_unaligned_le24(evt->dram.cvme_count));
960 else
961 WARN_ON_ONCE(evt->dram.media_hdr.type &
962 CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE);
963
964 trace_cxl_dram(cxlmd, type, cxlr, hpa, hpa_alias,
965 &evt->dram);
966 }
967 }
968 }
969 EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, "CXL");
970
__cxl_event_trace_record(const struct cxl_memdev * cxlmd,enum cxl_event_log_type type,struct cxl_event_record_raw * record)971 static void __cxl_event_trace_record(const struct cxl_memdev *cxlmd,
972 enum cxl_event_log_type type,
973 struct cxl_event_record_raw *record)
974 {
975 enum cxl_event_type ev_type = CXL_CPER_EVENT_GENERIC;
976 const uuid_t *uuid = &record->id;
977
978 if (uuid_equal(uuid, &CXL_EVENT_GEN_MEDIA_UUID))
979 ev_type = CXL_CPER_EVENT_GEN_MEDIA;
980 else if (uuid_equal(uuid, &CXL_EVENT_DRAM_UUID))
981 ev_type = CXL_CPER_EVENT_DRAM;
982 else if (uuid_equal(uuid, &CXL_EVENT_MEM_MODULE_UUID))
983 ev_type = CXL_CPER_EVENT_MEM_MODULE;
984 else if (uuid_equal(uuid, &CXL_EVENT_MEM_SPARING_UUID))
985 ev_type = CXL_CPER_EVENT_MEM_SPARING;
986
987 cxl_event_trace_record(cxlmd, type, ev_type, uuid, &record->event);
988 }
989
cxl_clear_event_record(struct cxl_memdev_state * mds,enum cxl_event_log_type log,struct cxl_get_event_payload * get_pl)990 static int cxl_clear_event_record(struct cxl_memdev_state *mds,
991 enum cxl_event_log_type log,
992 struct cxl_get_event_payload *get_pl)
993 {
994 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
995 struct cxl_mbox_clear_event_payload *payload;
996 u16 total = le16_to_cpu(get_pl->record_count);
997 u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;
998 size_t pl_size = struct_size(payload, handles, max_handles);
999 struct cxl_mbox_cmd mbox_cmd;
1000 u16 cnt;
1001 int rc = 0;
1002 int i;
1003
1004 /* Payload size may limit the max handles */
1005 if (pl_size > cxl_mbox->payload_size) {
1006 max_handles = (cxl_mbox->payload_size - sizeof(*payload)) /
1007 sizeof(__le16);
1008 pl_size = struct_size(payload, handles, max_handles);
1009 }
1010
1011 payload = kvzalloc(pl_size, GFP_KERNEL);
1012 if (!payload)
1013 return -ENOMEM;
1014
1015 *payload = (struct cxl_mbox_clear_event_payload) {
1016 .event_log = log,
1017 };
1018
1019 mbox_cmd = (struct cxl_mbox_cmd) {
1020 .opcode = CXL_MBOX_OP_CLEAR_EVENT_RECORD,
1021 .payload_in = payload,
1022 .size_in = pl_size,
1023 };
1024
1025 /*
1026 * Clear Event Records uses u8 for the handle cnt while Get Event
1027 * Record can return up to 0xffff records.
1028 */
1029 i = 0;
1030 for (cnt = 0; cnt < total; cnt++) {
1031 struct cxl_event_record_raw *raw = &get_pl->records[cnt];
1032 struct cxl_event_generic *gen = &raw->event.generic;
1033
1034 payload->handles[i++] = gen->hdr.handle;
1035 dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log,
1036 le16_to_cpu(payload->handles[i - 1]));
1037
1038 if (i == max_handles) {
1039 payload->nr_recs = i;
1040 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1041 if (rc)
1042 goto free_pl;
1043 i = 0;
1044 }
1045 }
1046
1047 /* Clear what is left if any */
1048 if (i) {
1049 payload->nr_recs = i;
1050 mbox_cmd.size_in = struct_size(payload, handles, i);
1051 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1052 if (rc)
1053 goto free_pl;
1054 }
1055
1056 free_pl:
1057 kvfree(payload);
1058 return rc;
1059 }
1060
cxl_mem_get_records_log(struct cxl_memdev_state * mds,enum cxl_event_log_type type)1061 static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
1062 enum cxl_event_log_type type)
1063 {
1064 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1065 struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
1066 struct device *dev = mds->cxlds.dev;
1067 struct cxl_get_event_payload *payload;
1068 u8 log_type = type;
1069 u16 nr_rec;
1070
1071 mutex_lock(&mds->event.log_lock);
1072 payload = mds->event.buf;
1073
1074 do {
1075 int rc, i;
1076 struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd) {
1077 .opcode = CXL_MBOX_OP_GET_EVENT_RECORD,
1078 .payload_in = &log_type,
1079 .size_in = sizeof(log_type),
1080 .payload_out = payload,
1081 .size_out = cxl_mbox->payload_size,
1082 .min_out = struct_size(payload, records, 0),
1083 };
1084
1085 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1086 if (rc) {
1087 dev_err_ratelimited(dev,
1088 "Event log '%d': Failed to query event records : %d",
1089 type, rc);
1090 break;
1091 }
1092
1093 nr_rec = le16_to_cpu(payload->record_count);
1094 if (!nr_rec)
1095 break;
1096
1097 for (i = 0; i < nr_rec; i++)
1098 __cxl_event_trace_record(cxlmd, type,
1099 &payload->records[i]);
1100
1101 if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW)
1102 trace_cxl_overflow(cxlmd, type, payload);
1103
1104 rc = cxl_clear_event_record(mds, type, payload);
1105 if (rc) {
1106 dev_err_ratelimited(dev,
1107 "Event log '%d': Failed to clear events : %d",
1108 type, rc);
1109 break;
1110 }
1111 } while (nr_rec);
1112
1113 mutex_unlock(&mds->event.log_lock);
1114 }
1115
1116 /**
1117 * cxl_mem_get_event_records - Get Event Records from the device
1118 * @mds: The driver data for the operation
1119 * @status: Event Status register value identifying which events are available.
1120 *
1121 * Retrieve all event records available on the device, report them as trace
1122 * events, and clear them.
1123 *
1124 * See CXL rev 3.0 @8.2.9.2.2 Get Event Records
1125 * See CXL rev 3.0 @8.2.9.2.3 Clear Event Records
1126 */
cxl_mem_get_event_records(struct cxl_memdev_state * mds,u32 status)1127 void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status)
1128 {
1129 dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status);
1130
1131 if (status & CXLDEV_EVENT_STATUS_FATAL)
1132 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL);
1133 if (status & CXLDEV_EVENT_STATUS_FAIL)
1134 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL);
1135 if (status & CXLDEV_EVENT_STATUS_WARN)
1136 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN);
1137 if (status & CXLDEV_EVENT_STATUS_INFO)
1138 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO);
1139 }
1140 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, "CXL");
1141
1142 /**
1143 * cxl_mem_get_partition_info - Get partition info
1144 * @mds: The driver data for the operation
1145 *
1146 * Retrieve the current partition info for the device specified. The active
1147 * values are the current capacity in bytes. If not 0, the 'next' values are
1148 * the pending values, in bytes, which take affect on next cold reset.
1149 *
1150 * Return: 0 if no error: or the result of the mailbox command.
1151 *
1152 * See CXL @8.2.9.5.2.1 Get Partition Info
1153 */
cxl_mem_get_partition_info(struct cxl_memdev_state * mds)1154 static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
1155 {
1156 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1157 struct cxl_mbox_get_partition_info pi;
1158 struct cxl_mbox_cmd mbox_cmd;
1159 int rc;
1160
1161 mbox_cmd = (struct cxl_mbox_cmd) {
1162 .opcode = CXL_MBOX_OP_GET_PARTITION_INFO,
1163 .size_out = sizeof(pi),
1164 .payload_out = &pi,
1165 };
1166 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1167 if (rc)
1168 return rc;
1169
1170 mds->active_volatile_bytes =
1171 le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1172 mds->active_persistent_bytes =
1173 le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;
1174
1175 return 0;
1176 }
1177
1178 /**
1179 * cxl_dev_state_identify() - Send the IDENTIFY command to the device.
1180 * @mds: The driver data for the operation
1181 *
1182 * Return: 0 if identify was executed successfully or media not ready.
1183 *
1184 * This will dispatch the identify command to the device and on success populate
1185 * structures to be exported to sysfs.
1186 */
cxl_dev_state_identify(struct cxl_memdev_state * mds)1187 int cxl_dev_state_identify(struct cxl_memdev_state *mds)
1188 {
1189 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1190 /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
1191 struct cxl_mbox_identify id;
1192 struct cxl_mbox_cmd mbox_cmd;
1193 u32 val;
1194 int rc;
1195
1196 if (!mds->cxlds.media_ready)
1197 return 0;
1198
1199 mbox_cmd = (struct cxl_mbox_cmd) {
1200 .opcode = CXL_MBOX_OP_IDENTIFY,
1201 .size_out = sizeof(id),
1202 .payload_out = &id,
1203 };
1204 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1205 if (rc < 0)
1206 return rc;
1207
1208 mds->total_bytes =
1209 le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;
1210 mds->volatile_only_bytes =
1211 le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;
1212 mds->persistent_only_bytes =
1213 le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;
1214 mds->partition_align_bytes =
1215 le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
1216
1217 mds->lsa_size = le32_to_cpu(id.lsa_size);
1218 memcpy(mds->firmware_version, id.fw_revision,
1219 sizeof(id.fw_revision));
1220
1221 if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) {
1222 val = get_unaligned_le24(id.poison_list_max_mer);
1223 mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);
1224 }
1225
1226 return 0;
1227 }
1228 EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, "CXL");
1229
__cxl_mem_sanitize(struct cxl_memdev_state * mds,u16 cmd)1230 static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
1231 {
1232 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1233 int rc;
1234 u32 sec_out = 0;
1235 struct cxl_get_security_output {
1236 __le32 flags;
1237 } out;
1238 struct cxl_mbox_cmd sec_cmd = {
1239 .opcode = CXL_MBOX_OP_GET_SECURITY_STATE,
1240 .payload_out = &out,
1241 .size_out = sizeof(out),
1242 };
1243 struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };
1244
1245 if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)
1246 return -EINVAL;
1247
1248 rc = cxl_internal_send_cmd(cxl_mbox, &sec_cmd);
1249 if (rc < 0) {
1250 dev_err(cxl_mbox->host, "Failed to get security state : %d", rc);
1251 return rc;
1252 }
1253
1254 /*
1255 * Prior to using these commands, any security applied to
1256 * the user data areas of the device shall be DISABLED (or
1257 * UNLOCKED for secure erase case).
1258 */
1259 sec_out = le32_to_cpu(out.flags);
1260 if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET)
1261 return -EINVAL;
1262
1263 if (cmd == CXL_MBOX_OP_SECURE_ERASE &&
1264 sec_out & CXL_PMEM_SEC_STATE_LOCKED)
1265 return -EINVAL;
1266
1267 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1268 if (rc < 0) {
1269 dev_err(cxl_mbox->host, "Failed to sanitize device : %d", rc);
1270 return rc;
1271 }
1272
1273 return 0;
1274 }
1275
1276
1277 /**
1278 * cxl_mem_sanitize() - Send a sanitization command to the device.
1279 * @cxlmd: The device for the operation
1280 * @cmd: The specific sanitization command opcode
1281 *
1282 * Return: 0 if the command was executed successfully, regardless of
1283 * whether or not the actual security operation is done in the background,
1284 * such as for the Sanitize case.
1285 * Error return values can be the result of the mailbox command, -EINVAL
1286 * when security requirements are not met or invalid contexts, or -EBUSY
1287 * if the sanitize operation is already in flight.
1288 *
1289 * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase.
1290 */
cxl_mem_sanitize(struct cxl_memdev * cxlmd,u16 cmd)1291 int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
1292 {
1293 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1294 struct cxl_port *endpoint;
1295
1296 /* synchronize with cxl_mem_probe() and decoder write operations */
1297 guard(device)(&cxlmd->dev);
1298 endpoint = cxlmd->endpoint;
1299 guard(rwsem_read)(&cxl_rwsem.region);
1300 /*
1301 * Require an endpoint to be safe otherwise the driver can not
1302 * be sure that the device is unmapped.
1303 */
1304 if (endpoint && cxl_num_decoders_committed(endpoint) == 0)
1305 return __cxl_mem_sanitize(mds, cmd);
1306
1307 return -EBUSY;
1308 }
1309
add_part(struct cxl_dpa_info * info,u64 start,u64 size,enum cxl_partition_mode mode)1310 static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode)
1311 {
1312 int i = info->nr_partitions;
1313
1314 if (size == 0)
1315 return;
1316
1317 info->part[i].range = (struct range) {
1318 .start = start,
1319 .end = start + size - 1,
1320 };
1321 info->part[i].mode = mode;
1322 info->nr_partitions++;
1323 }
1324
cxl_mem_dpa_fetch(struct cxl_memdev_state * mds,struct cxl_dpa_info * info)1325 int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
1326 {
1327 struct cxl_dev_state *cxlds = &mds->cxlds;
1328 struct device *dev = cxlds->dev;
1329 int rc;
1330
1331 if (!cxlds->media_ready) {
1332 info->size = 0;
1333 return 0;
1334 }
1335
1336 info->size = mds->total_bytes;
1337
1338 if (mds->partition_align_bytes == 0) {
1339 add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM);
1340 add_part(info, mds->volatile_only_bytes,
1341 mds->persistent_only_bytes, CXL_PARTMODE_PMEM);
1342 return 0;
1343 }
1344
1345 rc = cxl_mem_get_partition_info(mds);
1346 if (rc) {
1347 dev_err(dev, "Failed to query partition information\n");
1348 return rc;
1349 }
1350
1351 add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM);
1352 add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes,
1353 CXL_PARTMODE_PMEM);
1354
1355 return 0;
1356 }
1357 EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");
1358
cxl_get_dirty_count(struct cxl_memdev_state * mds,u32 * count)1359 int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count)
1360 {
1361 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1362 struct cxl_mbox_get_health_info_out hi;
1363 struct cxl_mbox_cmd mbox_cmd;
1364 int rc;
1365
1366 mbox_cmd = (struct cxl_mbox_cmd) {
1367 .opcode = CXL_MBOX_OP_GET_HEALTH_INFO,
1368 .size_out = sizeof(hi),
1369 .payload_out = &hi,
1370 };
1371
1372 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1373 if (!rc)
1374 *count = le32_to_cpu(hi.dirty_shutdown_cnt);
1375
1376 return rc;
1377 }
1378 EXPORT_SYMBOL_NS_GPL(cxl_get_dirty_count, "CXL");
1379
cxl_arm_dirty_shutdown(struct cxl_memdev_state * mds)1380 int cxl_arm_dirty_shutdown(struct cxl_memdev_state *mds)
1381 {
1382 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1383 struct cxl_mbox_cmd mbox_cmd;
1384 struct cxl_mbox_set_shutdown_state_in in = {
1385 .state = 1
1386 };
1387
1388 mbox_cmd = (struct cxl_mbox_cmd) {
1389 .opcode = CXL_MBOX_OP_SET_SHUTDOWN_STATE,
1390 .size_in = sizeof(in),
1391 .payload_in = &in,
1392 };
1393
1394 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1395 }
1396 EXPORT_SYMBOL_NS_GPL(cxl_arm_dirty_shutdown, "CXL");
1397
cxl_set_timestamp(struct cxl_memdev_state * mds)1398 int cxl_set_timestamp(struct cxl_memdev_state *mds)
1399 {
1400 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1401 struct cxl_mbox_cmd mbox_cmd;
1402 struct cxl_mbox_set_timestamp_in pi;
1403 int rc;
1404
1405 pi.timestamp = cpu_to_le64(ktime_get_real_ns());
1406 mbox_cmd = (struct cxl_mbox_cmd) {
1407 .opcode = CXL_MBOX_OP_SET_TIMESTAMP,
1408 .size_in = sizeof(pi),
1409 .payload_in = &pi,
1410 };
1411
1412 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1413 /*
1414 * Command is optional. Devices may have another way of providing
1415 * a timestamp, or may return all 0s in timestamp fields.
1416 * Don't report an error if this command isn't supported
1417 */
1418 if (rc && (mbox_cmd.return_code != CXL_MBOX_CMD_RC_UNSUPPORTED))
1419 return rc;
1420
1421 return 0;
1422 }
1423 EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, "CXL");
1424
cxl_mem_get_poison(struct cxl_memdev * cxlmd,u64 offset,u64 len,struct cxl_region * cxlr)1425 int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
1426 struct cxl_region *cxlr)
1427 {
1428 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1429 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
1430 struct cxl_mbox_poison_out *po;
1431 struct cxl_mbox_poison_in pi;
1432 int nr_records = 0;
1433 int rc;
1434
1435 ACQUIRE(mutex_intr, lock)(&mds->poison.mutex);
1436 if ((rc = ACQUIRE_ERR(mutex_intr, &lock)))
1437 return rc;
1438
1439 po = mds->poison.list_out;
1440 pi.offset = cpu_to_le64(offset);
1441 pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT);
1442
1443 do {
1444 struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd){
1445 .opcode = CXL_MBOX_OP_GET_POISON,
1446 .size_in = sizeof(pi),
1447 .payload_in = &pi,
1448 .size_out = cxl_mbox->payload_size,
1449 .payload_out = po,
1450 .min_out = struct_size(po, record, 0),
1451 };
1452
1453 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1454 if (rc)
1455 break;
1456
1457 for (int i = 0; i < le16_to_cpu(po->count); i++)
1458 trace_cxl_poison(cxlmd, cxlr, &po->record[i],
1459 po->flags, po->overflow_ts,
1460 CXL_POISON_TRACE_LIST);
1461
1462 /* Protect against an uncleared _FLAG_MORE */
1463 nr_records = nr_records + le16_to_cpu(po->count);
1464 if (nr_records >= mds->poison.max_errors) {
1465 dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n",
1466 nr_records);
1467 break;
1468 }
1469 } while (po->flags & CXL_POISON_FLAG_MORE);
1470
1471 return rc;
1472 }
1473 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, "CXL");
1474
free_poison_buf(void * buf)1475 static void free_poison_buf(void *buf)
1476 {
1477 kvfree(buf);
1478 }
1479
1480 /* Get Poison List output buffer is protected by mds->poison.lock */
cxl_poison_alloc_buf(struct cxl_memdev_state * mds)1481 static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
1482 {
1483 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1484
1485 mds->poison.list_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
1486 if (!mds->poison.list_out)
1487 return -ENOMEM;
1488
1489 return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf,
1490 mds->poison.list_out);
1491 }
1492
cxl_poison_state_init(struct cxl_memdev_state * mds)1493 int cxl_poison_state_init(struct cxl_memdev_state *mds)
1494 {
1495 int rc;
1496
1497 if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds))
1498 return 0;
1499
1500 rc = cxl_poison_alloc_buf(mds);
1501 if (rc) {
1502 clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds);
1503 return rc;
1504 }
1505
1506 mutex_init(&mds->poison.mutex);
1507 return 0;
1508 }
1509 EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, "CXL");
1510
cxl_mailbox_init(struct cxl_mailbox * cxl_mbox,struct device * host)1511 int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host)
1512 {
1513 if (!cxl_mbox || !host)
1514 return -EINVAL;
1515
1516 cxl_mbox->host = host;
1517 mutex_init(&cxl_mbox->mbox_mutex);
1518 rcuwait_init(&cxl_mbox->mbox_wait);
1519
1520 return 0;
1521 }
1522 EXPORT_SYMBOL_NS_GPL(cxl_mailbox_init, "CXL");
1523
cxl_memdev_state_create(struct device * dev)1524 struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
1525 {
1526 struct cxl_memdev_state *mds;
1527 int rc;
1528
1529 mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
1530 if (!mds) {
1531 dev_err(dev, "No memory available\n");
1532 return ERR_PTR(-ENOMEM);
1533 }
1534
1535 mutex_init(&mds->event.log_lock);
1536 mds->cxlds.dev = dev;
1537 mds->cxlds.reg_map.host = dev;
1538 mds->cxlds.cxl_mbox.host = dev;
1539 mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE;
1540 mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
1541
1542 rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier);
1543 if (rc == -EOPNOTSUPP)
1544 dev_warn(dev, "CXL MCE unsupported\n");
1545 else if (rc)
1546 return ERR_PTR(rc);
1547
1548 return mds;
1549 }
1550 EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, "CXL");
1551
cxl_mbox_init(void)1552 void __init cxl_mbox_init(void)
1553 {
1554 struct dentry *mbox_debugfs;
1555
1556 mbox_debugfs = cxl_debugfs_create_dir("mbox");
1557 debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
1558 &cxl_raw_allow_all);
1559 }
1560