1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Secure Encrypted Virtualization (SEV) interface
4 *
5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
6 *
7 * Author: Brijesh Singh <brijesh.singh@amd.com>
8 */
9
10 #include <linux/bitfield.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/kthread.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/spinlock.h>
17 #include <linux/spinlock_types.h>
18 #include <linux/types.h>
19 #include <linux/mutex.h>
20 #include <linux/delay.h>
21 #include <linux/hw_random.h>
22 #include <linux/ccp.h>
23 #include <linux/firmware.h>
24 #include <linux/panic_notifier.h>
25 #include <linux/gfp.h>
26 #include <linux/cpufeature.h>
27 #include <linux/fs.h>
28 #include <linux/fs_struct.h>
29 #include <linux/psp.h>
30 #include <linux/amd-iommu.h>
31
32 #include <asm/smp.h>
33 #include <asm/cacheflush.h>
34 #include <asm/e820/types.h>
35 #include <asm/sev.h>
36
37 #include "psp-dev.h"
38 #include "sev-dev.h"
39
40 #define DEVICE_NAME "sev"
41 #define SEV_FW_FILE "amd/sev.fw"
42 #define SEV_FW_NAME_SIZE 64
43
44 /* Minimum firmware version required for the SEV-SNP support */
45 #define SNP_MIN_API_MAJOR 1
46 #define SNP_MIN_API_MINOR 51
47
48 /*
49 * Maximum number of firmware-writable buffers that might be specified
50 * in the parameters of a legacy SEV command buffer.
51 */
52 #define CMD_BUF_FW_WRITABLE_MAX 2
53
54 /* Leave room in the descriptor array for an end-of-list indicator. */
55 #define CMD_BUF_DESC_MAX (CMD_BUF_FW_WRITABLE_MAX + 1)
56
57 static DEFINE_MUTEX(sev_cmd_mutex);
58 static struct sev_misc_dev *misc_dev;
59
60 static int psp_cmd_timeout = 100;
61 module_param(psp_cmd_timeout, int, 0644);
62 MODULE_PARM_DESC(psp_cmd_timeout, " default timeout value, in seconds, for PSP commands");
63
64 static int psp_probe_timeout = 5;
65 module_param(psp_probe_timeout, int, 0644);
66 MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
67
68 static char *init_ex_path;
69 module_param(init_ex_path, charp, 0444);
70 MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
71
72 static bool psp_init_on_probe = true;
73 module_param(psp_init_on_probe, bool, 0444);
74 MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
75
76 MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
77 MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
78 MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
79 MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */
80
81 static bool psp_dead;
82 static int psp_timeout;
83
84 /* Trusted Memory Region (TMR):
85 * The TMR is a 1MB area that must be 1MB aligned. Use the page allocator
86 * to allocate the memory, which will return aligned memory for the specified
87 * allocation order.
88 *
89 * When SEV-SNP is enabled the TMR needs to be 2MB aligned and 2MB sized.
90 */
91 #define SEV_TMR_SIZE (1024 * 1024)
92 #define SNP_TMR_SIZE (2 * 1024 * 1024)
93
94 static void *sev_es_tmr;
95 static size_t sev_es_tmr_size = SEV_TMR_SIZE;
96
97 /* INIT_EX NV Storage:
98 * The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page
99 * allocator to allocate the memory, which will return aligned memory for the
100 * specified allocation order.
101 */
102 #define NV_LENGTH (32 * 1024)
103 static void *sev_init_ex_buffer;
104
105 /*
106 * SEV_DATA_RANGE_LIST:
107 * Array containing range of pages that firmware transitions to HV-fixed
108 * page state.
109 */
110 static struct sev_data_range_list *snp_range_list;
111
sev_version_greater_or_equal(u8 maj,u8 min)112 static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
113 {
114 struct sev_device *sev = psp_master->sev_data;
115
116 if (sev->api_major > maj)
117 return true;
118
119 if (sev->api_major == maj && sev->api_minor >= min)
120 return true;
121
122 return false;
123 }
124
sev_irq_handler(int irq,void * data,unsigned int status)125 static void sev_irq_handler(int irq, void *data, unsigned int status)
126 {
127 struct sev_device *sev = data;
128 int reg;
129
130 /* Check if it is command completion: */
131 if (!(status & SEV_CMD_COMPLETE))
132 return;
133
134 /* Check if it is SEV command completion: */
135 reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
136 if (FIELD_GET(PSP_CMDRESP_RESP, reg)) {
137 sev->int_rcvd = 1;
138 wake_up(&sev->int_queue);
139 }
140 }
141
sev_wait_cmd_ioc(struct sev_device * sev,unsigned int * reg,unsigned int timeout)142 static int sev_wait_cmd_ioc(struct sev_device *sev,
143 unsigned int *reg, unsigned int timeout)
144 {
145 int ret;
146
147 /*
148 * If invoked during panic handling, local interrupts are disabled,
149 * so the PSP command completion interrupt can't be used. Poll for
150 * PSP command completion instead.
151 */
152 if (irqs_disabled()) {
153 unsigned long timeout_usecs = (timeout * USEC_PER_SEC) / 10;
154
155 /* Poll for SEV command completion: */
156 while (timeout_usecs--) {
157 *reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
158 if (*reg & PSP_CMDRESP_RESP)
159 return 0;
160
161 udelay(10);
162 }
163 return -ETIMEDOUT;
164 }
165
166 ret = wait_event_timeout(sev->int_queue,
167 sev->int_rcvd, timeout * HZ);
168 if (!ret)
169 return -ETIMEDOUT;
170
171 *reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
172
173 return 0;
174 }
175
sev_cmd_buffer_len(int cmd)176 static int sev_cmd_buffer_len(int cmd)
177 {
178 switch (cmd) {
179 case SEV_CMD_INIT: return sizeof(struct sev_data_init);
180 case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex);
181 case SEV_CMD_SNP_SHUTDOWN_EX: return sizeof(struct sev_data_snp_shutdown_ex);
182 case SEV_CMD_SNP_INIT_EX: return sizeof(struct sev_data_snp_init_ex);
183 case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status);
184 case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr);
185 case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import);
186 case SEV_CMD_PDH_CERT_EXPORT: return sizeof(struct sev_data_pdh_cert_export);
187 case SEV_CMD_LAUNCH_START: return sizeof(struct sev_data_launch_start);
188 case SEV_CMD_LAUNCH_UPDATE_DATA: return sizeof(struct sev_data_launch_update_data);
189 case SEV_CMD_LAUNCH_UPDATE_VMSA: return sizeof(struct sev_data_launch_update_vmsa);
190 case SEV_CMD_LAUNCH_FINISH: return sizeof(struct sev_data_launch_finish);
191 case SEV_CMD_LAUNCH_MEASURE: return sizeof(struct sev_data_launch_measure);
192 case SEV_CMD_ACTIVATE: return sizeof(struct sev_data_activate);
193 case SEV_CMD_DEACTIVATE: return sizeof(struct sev_data_deactivate);
194 case SEV_CMD_DECOMMISSION: return sizeof(struct sev_data_decommission);
195 case SEV_CMD_GUEST_STATUS: return sizeof(struct sev_data_guest_status);
196 case SEV_CMD_DBG_DECRYPT: return sizeof(struct sev_data_dbg);
197 case SEV_CMD_DBG_ENCRYPT: return sizeof(struct sev_data_dbg);
198 case SEV_CMD_SEND_START: return sizeof(struct sev_data_send_start);
199 case SEV_CMD_SEND_UPDATE_DATA: return sizeof(struct sev_data_send_update_data);
200 case SEV_CMD_SEND_UPDATE_VMSA: return sizeof(struct sev_data_send_update_vmsa);
201 case SEV_CMD_SEND_FINISH: return sizeof(struct sev_data_send_finish);
202 case SEV_CMD_RECEIVE_START: return sizeof(struct sev_data_receive_start);
203 case SEV_CMD_RECEIVE_FINISH: return sizeof(struct sev_data_receive_finish);
204 case SEV_CMD_RECEIVE_UPDATE_DATA: return sizeof(struct sev_data_receive_update_data);
205 case SEV_CMD_RECEIVE_UPDATE_VMSA: return sizeof(struct sev_data_receive_update_vmsa);
206 case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret);
207 case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware);
208 case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id);
209 case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report);
210 case SEV_CMD_SEND_CANCEL: return sizeof(struct sev_data_send_cancel);
211 case SEV_CMD_SNP_GCTX_CREATE: return sizeof(struct sev_data_snp_addr);
212 case SEV_CMD_SNP_LAUNCH_START: return sizeof(struct sev_data_snp_launch_start);
213 case SEV_CMD_SNP_LAUNCH_UPDATE: return sizeof(struct sev_data_snp_launch_update);
214 case SEV_CMD_SNP_ACTIVATE: return sizeof(struct sev_data_snp_activate);
215 case SEV_CMD_SNP_DECOMMISSION: return sizeof(struct sev_data_snp_addr);
216 case SEV_CMD_SNP_PAGE_RECLAIM: return sizeof(struct sev_data_snp_page_reclaim);
217 case SEV_CMD_SNP_GUEST_STATUS: return sizeof(struct sev_data_snp_guest_status);
218 case SEV_CMD_SNP_LAUNCH_FINISH: return sizeof(struct sev_data_snp_launch_finish);
219 case SEV_CMD_SNP_DBG_DECRYPT: return sizeof(struct sev_data_snp_dbg);
220 case SEV_CMD_SNP_DBG_ENCRYPT: return sizeof(struct sev_data_snp_dbg);
221 case SEV_CMD_SNP_PAGE_UNSMASH: return sizeof(struct sev_data_snp_page_unsmash);
222 case SEV_CMD_SNP_PLATFORM_STATUS: return sizeof(struct sev_data_snp_addr);
223 case SEV_CMD_SNP_GUEST_REQUEST: return sizeof(struct sev_data_snp_guest_request);
224 case SEV_CMD_SNP_CONFIG: return sizeof(struct sev_user_data_snp_config);
225 case SEV_CMD_SNP_COMMIT: return sizeof(struct sev_data_snp_commit);
226 default: return 0;
227 }
228
229 return 0;
230 }
231
open_file_as_root(const char * filename,int flags,umode_t mode)232 static struct file *open_file_as_root(const char *filename, int flags, umode_t mode)
233 {
234 struct file *fp;
235 struct path root;
236 struct cred *cred;
237 const struct cred *old_cred;
238
239 task_lock(&init_task);
240 get_fs_root(init_task.fs, &root);
241 task_unlock(&init_task);
242
243 cred = prepare_creds();
244 if (!cred)
245 return ERR_PTR(-ENOMEM);
246 cred->fsuid = GLOBAL_ROOT_UID;
247 old_cred = override_creds(cred);
248
249 fp = file_open_root(&root, filename, flags, mode);
250 path_put(&root);
251
252 revert_creds(old_cred);
253
254 return fp;
255 }
256
sev_read_init_ex_file(void)257 static int sev_read_init_ex_file(void)
258 {
259 struct sev_device *sev = psp_master->sev_data;
260 struct file *fp;
261 ssize_t nread;
262
263 lockdep_assert_held(&sev_cmd_mutex);
264
265 if (!sev_init_ex_buffer)
266 return -EOPNOTSUPP;
267
268 fp = open_file_as_root(init_ex_path, O_RDONLY, 0);
269 if (IS_ERR(fp)) {
270 int ret = PTR_ERR(fp);
271
272 if (ret == -ENOENT) {
273 dev_info(sev->dev,
274 "SEV: %s does not exist and will be created later.\n",
275 init_ex_path);
276 ret = 0;
277 } else {
278 dev_err(sev->dev,
279 "SEV: could not open %s for read, error %d\n",
280 init_ex_path, ret);
281 }
282 return ret;
283 }
284
285 nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
286 if (nread != NV_LENGTH) {
287 dev_info(sev->dev,
288 "SEV: could not read %u bytes to non volatile memory area, ret %ld\n",
289 NV_LENGTH, nread);
290 }
291
292 dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
293 filp_close(fp, NULL);
294
295 return 0;
296 }
297
sev_write_init_ex_file(void)298 static int sev_write_init_ex_file(void)
299 {
300 struct sev_device *sev = psp_master->sev_data;
301 struct file *fp;
302 loff_t offset = 0;
303 ssize_t nwrite;
304
305 lockdep_assert_held(&sev_cmd_mutex);
306
307 if (!sev_init_ex_buffer)
308 return 0;
309
310 fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600);
311 if (IS_ERR(fp)) {
312 int ret = PTR_ERR(fp);
313
314 dev_err(sev->dev,
315 "SEV: could not open file for write, error %d\n",
316 ret);
317 return ret;
318 }
319
320 nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
321 vfs_fsync(fp, 0);
322 filp_close(fp, NULL);
323
324 if (nwrite != NV_LENGTH) {
325 dev_err(sev->dev,
326 "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
327 NV_LENGTH, nwrite);
328 return -EIO;
329 }
330
331 dev_dbg(sev->dev, "SEV: write successful to NV file\n");
332
333 return 0;
334 }
335
sev_write_init_ex_file_if_required(int cmd_id)336 static int sev_write_init_ex_file_if_required(int cmd_id)
337 {
338 lockdep_assert_held(&sev_cmd_mutex);
339
340 if (!sev_init_ex_buffer)
341 return 0;
342
343 /*
344 * Only a few platform commands modify the SPI/NV area, but none of the
345 * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
346 * PEK_CERT_IMPORT, and PDH_GEN do.
347 */
348 switch (cmd_id) {
349 case SEV_CMD_FACTORY_RESET:
350 case SEV_CMD_INIT_EX:
351 case SEV_CMD_PDH_GEN:
352 case SEV_CMD_PEK_CERT_IMPORT:
353 case SEV_CMD_PEK_GEN:
354 break;
355 default:
356 return 0;
357 }
358
359 return sev_write_init_ex_file();
360 }
361
362 /*
363 * snp_reclaim_pages() needs __sev_do_cmd_locked(), and __sev_do_cmd_locked()
364 * needs snp_reclaim_pages(), so a forward declaration is needed.
365 */
366 static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret);
367
snp_reclaim_pages(unsigned long paddr,unsigned int npages,bool locked)368 static int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked)
369 {
370 int ret, err, i;
371
372 paddr = __sme_clr(ALIGN_DOWN(paddr, PAGE_SIZE));
373
374 for (i = 0; i < npages; i++, paddr += PAGE_SIZE) {
375 struct sev_data_snp_page_reclaim data = {0};
376
377 data.paddr = paddr;
378
379 if (locked)
380 ret = __sev_do_cmd_locked(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
381 else
382 ret = sev_do_cmd(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
383
384 if (ret)
385 goto cleanup;
386
387 ret = rmp_make_shared(__phys_to_pfn(paddr), PG_LEVEL_4K);
388 if (ret)
389 goto cleanup;
390 }
391
392 return 0;
393
394 cleanup:
395 /*
396 * If there was a failure reclaiming the page then it is no longer safe
397 * to release it back to the system; leak it instead.
398 */
399 snp_leak_pages(__phys_to_pfn(paddr), npages - i);
400 return ret;
401 }
402
rmp_mark_pages_firmware(unsigned long paddr,unsigned int npages,bool locked)403 static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked)
404 {
405 unsigned long pfn = __sme_clr(paddr) >> PAGE_SHIFT;
406 int rc, i;
407
408 for (i = 0; i < npages; i++, pfn++) {
409 rc = rmp_make_private(pfn, 0, PG_LEVEL_4K, 0, true);
410 if (rc)
411 goto cleanup;
412 }
413
414 return 0;
415
416 cleanup:
417 /*
418 * Try unrolling the firmware state changes by
419 * reclaiming the pages which were already changed to the
420 * firmware state.
421 */
422 snp_reclaim_pages(paddr, i, locked);
423
424 return rc;
425 }
426
__snp_alloc_firmware_pages(gfp_t gfp_mask,int order)427 static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order)
428 {
429 unsigned long npages = 1ul << order, paddr;
430 struct sev_device *sev;
431 struct page *page;
432
433 if (!psp_master || !psp_master->sev_data)
434 return NULL;
435
436 page = alloc_pages(gfp_mask, order);
437 if (!page)
438 return NULL;
439
440 /* If SEV-SNP is initialized then add the page in RMP table. */
441 sev = psp_master->sev_data;
442 if (!sev->snp_initialized)
443 return page;
444
445 paddr = __pa((unsigned long)page_address(page));
446 if (rmp_mark_pages_firmware(paddr, npages, false))
447 return NULL;
448
449 return page;
450 }
451
snp_alloc_firmware_page(gfp_t gfp_mask)452 void *snp_alloc_firmware_page(gfp_t gfp_mask)
453 {
454 struct page *page;
455
456 page = __snp_alloc_firmware_pages(gfp_mask, 0);
457
458 return page ? page_address(page) : NULL;
459 }
460 EXPORT_SYMBOL_GPL(snp_alloc_firmware_page);
461
__snp_free_firmware_pages(struct page * page,int order,bool locked)462 static void __snp_free_firmware_pages(struct page *page, int order, bool locked)
463 {
464 struct sev_device *sev = psp_master->sev_data;
465 unsigned long paddr, npages = 1ul << order;
466
467 if (!page)
468 return;
469
470 paddr = __pa((unsigned long)page_address(page));
471 if (sev->snp_initialized &&
472 snp_reclaim_pages(paddr, npages, locked))
473 return;
474
475 __free_pages(page, order);
476 }
477
snp_free_firmware_page(void * addr)478 void snp_free_firmware_page(void *addr)
479 {
480 if (!addr)
481 return;
482
483 __snp_free_firmware_pages(virt_to_page(addr), 0, false);
484 }
485 EXPORT_SYMBOL_GPL(snp_free_firmware_page);
486
sev_fw_alloc(unsigned long len)487 static void *sev_fw_alloc(unsigned long len)
488 {
489 struct page *page;
490
491 page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len));
492 if (!page)
493 return NULL;
494
495 return page_address(page);
496 }
497
498 /**
499 * struct cmd_buf_desc - descriptors for managing legacy SEV command address
500 * parameters corresponding to buffers that may be written to by firmware.
501 *
502 * @paddr_ptr: pointer to the address parameter in the command buffer which may
503 * need to be saved/restored depending on whether a bounce buffer
504 * is used. In the case of a bounce buffer, the command buffer
505 * needs to be updated with the address of the new bounce buffer
506 * snp_map_cmd_buf_desc() has allocated specifically for it. Must
507 * be NULL if this descriptor is only an end-of-list indicator.
508 *
509 * @paddr_orig: storage for the original address parameter, which can be used to
510 * restore the original value in @paddr_ptr in cases where it is
511 * replaced with the address of a bounce buffer.
512 *
513 * @len: length of buffer located at the address originally stored at @paddr_ptr
514 *
515 * @guest_owned: true if the address corresponds to guest-owned pages, in which
516 * case bounce buffers are not needed.
517 */
518 struct cmd_buf_desc {
519 u64 *paddr_ptr;
520 u64 paddr_orig;
521 u32 len;
522 bool guest_owned;
523 };
524
525 /*
526 * If a legacy SEV command parameter is a memory address, those pages in
527 * turn need to be transitioned to/from firmware-owned before/after
528 * executing the firmware command.
529 *
530 * Additionally, in cases where those pages are not guest-owned, a bounce
531 * buffer is needed in place of the original memory address parameter.
532 *
533 * A set of descriptors are used to keep track of this handling, and
534 * initialized here based on the specific commands being executed.
535 */
snp_populate_cmd_buf_desc_list(int cmd,void * cmd_buf,struct cmd_buf_desc * desc_list)536 static void snp_populate_cmd_buf_desc_list(int cmd, void *cmd_buf,
537 struct cmd_buf_desc *desc_list)
538 {
539 switch (cmd) {
540 case SEV_CMD_PDH_CERT_EXPORT: {
541 struct sev_data_pdh_cert_export *data = cmd_buf;
542
543 desc_list[0].paddr_ptr = &data->pdh_cert_address;
544 desc_list[0].len = data->pdh_cert_len;
545 desc_list[1].paddr_ptr = &data->cert_chain_address;
546 desc_list[1].len = data->cert_chain_len;
547 break;
548 }
549 case SEV_CMD_GET_ID: {
550 struct sev_data_get_id *data = cmd_buf;
551
552 desc_list[0].paddr_ptr = &data->address;
553 desc_list[0].len = data->len;
554 break;
555 }
556 case SEV_CMD_PEK_CSR: {
557 struct sev_data_pek_csr *data = cmd_buf;
558
559 desc_list[0].paddr_ptr = &data->address;
560 desc_list[0].len = data->len;
561 break;
562 }
563 case SEV_CMD_LAUNCH_UPDATE_DATA: {
564 struct sev_data_launch_update_data *data = cmd_buf;
565
566 desc_list[0].paddr_ptr = &data->address;
567 desc_list[0].len = data->len;
568 desc_list[0].guest_owned = true;
569 break;
570 }
571 case SEV_CMD_LAUNCH_UPDATE_VMSA: {
572 struct sev_data_launch_update_vmsa *data = cmd_buf;
573
574 desc_list[0].paddr_ptr = &data->address;
575 desc_list[0].len = data->len;
576 desc_list[0].guest_owned = true;
577 break;
578 }
579 case SEV_CMD_LAUNCH_MEASURE: {
580 struct sev_data_launch_measure *data = cmd_buf;
581
582 desc_list[0].paddr_ptr = &data->address;
583 desc_list[0].len = data->len;
584 break;
585 }
586 case SEV_CMD_LAUNCH_UPDATE_SECRET: {
587 struct sev_data_launch_secret *data = cmd_buf;
588
589 desc_list[0].paddr_ptr = &data->guest_address;
590 desc_list[0].len = data->guest_len;
591 desc_list[0].guest_owned = true;
592 break;
593 }
594 case SEV_CMD_DBG_DECRYPT: {
595 struct sev_data_dbg *data = cmd_buf;
596
597 desc_list[0].paddr_ptr = &data->dst_addr;
598 desc_list[0].len = data->len;
599 desc_list[0].guest_owned = true;
600 break;
601 }
602 case SEV_CMD_DBG_ENCRYPT: {
603 struct sev_data_dbg *data = cmd_buf;
604
605 desc_list[0].paddr_ptr = &data->dst_addr;
606 desc_list[0].len = data->len;
607 desc_list[0].guest_owned = true;
608 break;
609 }
610 case SEV_CMD_ATTESTATION_REPORT: {
611 struct sev_data_attestation_report *data = cmd_buf;
612
613 desc_list[0].paddr_ptr = &data->address;
614 desc_list[0].len = data->len;
615 break;
616 }
617 case SEV_CMD_SEND_START: {
618 struct sev_data_send_start *data = cmd_buf;
619
620 desc_list[0].paddr_ptr = &data->session_address;
621 desc_list[0].len = data->session_len;
622 break;
623 }
624 case SEV_CMD_SEND_UPDATE_DATA: {
625 struct sev_data_send_update_data *data = cmd_buf;
626
627 desc_list[0].paddr_ptr = &data->hdr_address;
628 desc_list[0].len = data->hdr_len;
629 desc_list[1].paddr_ptr = &data->trans_address;
630 desc_list[1].len = data->trans_len;
631 break;
632 }
633 case SEV_CMD_SEND_UPDATE_VMSA: {
634 struct sev_data_send_update_vmsa *data = cmd_buf;
635
636 desc_list[0].paddr_ptr = &data->hdr_address;
637 desc_list[0].len = data->hdr_len;
638 desc_list[1].paddr_ptr = &data->trans_address;
639 desc_list[1].len = data->trans_len;
640 break;
641 }
642 case SEV_CMD_RECEIVE_UPDATE_DATA: {
643 struct sev_data_receive_update_data *data = cmd_buf;
644
645 desc_list[0].paddr_ptr = &data->guest_address;
646 desc_list[0].len = data->guest_len;
647 desc_list[0].guest_owned = true;
648 break;
649 }
650 case SEV_CMD_RECEIVE_UPDATE_VMSA: {
651 struct sev_data_receive_update_vmsa *data = cmd_buf;
652
653 desc_list[0].paddr_ptr = &data->guest_address;
654 desc_list[0].len = data->guest_len;
655 desc_list[0].guest_owned = true;
656 break;
657 }
658 default:
659 break;
660 }
661 }
662
snp_map_cmd_buf_desc(struct cmd_buf_desc * desc)663 static int snp_map_cmd_buf_desc(struct cmd_buf_desc *desc)
664 {
665 unsigned int npages;
666
667 if (!desc->len)
668 return 0;
669
670 /* Allocate a bounce buffer if this isn't a guest owned page. */
671 if (!desc->guest_owned) {
672 struct page *page;
673
674 page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(desc->len));
675 if (!page) {
676 pr_warn("Failed to allocate bounce buffer for SEV legacy command.\n");
677 return -ENOMEM;
678 }
679
680 desc->paddr_orig = *desc->paddr_ptr;
681 *desc->paddr_ptr = __psp_pa(page_to_virt(page));
682 }
683
684 npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
685
686 /* Transition the buffer to firmware-owned. */
687 if (rmp_mark_pages_firmware(*desc->paddr_ptr, npages, true)) {
688 pr_warn("Error moving pages to firmware-owned state for SEV legacy command.\n");
689 return -EFAULT;
690 }
691
692 return 0;
693 }
694
snp_unmap_cmd_buf_desc(struct cmd_buf_desc * desc)695 static int snp_unmap_cmd_buf_desc(struct cmd_buf_desc *desc)
696 {
697 unsigned int npages;
698
699 if (!desc->len)
700 return 0;
701
702 npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
703
704 /* Transition the buffers back to hypervisor-owned. */
705 if (snp_reclaim_pages(*desc->paddr_ptr, npages, true)) {
706 pr_warn("Failed to reclaim firmware-owned pages while issuing SEV legacy command.\n");
707 return -EFAULT;
708 }
709
710 /* Copy data from bounce buffer and then free it. */
711 if (!desc->guest_owned) {
712 void *bounce_buf = __va(__sme_clr(*desc->paddr_ptr));
713 void *dst_buf = __va(__sme_clr(desc->paddr_orig));
714
715 memcpy(dst_buf, bounce_buf, desc->len);
716 __free_pages(virt_to_page(bounce_buf), get_order(desc->len));
717
718 /* Restore the original address in the command buffer. */
719 *desc->paddr_ptr = desc->paddr_orig;
720 }
721
722 return 0;
723 }
724
snp_map_cmd_buf_desc_list(int cmd,void * cmd_buf,struct cmd_buf_desc * desc_list)725 static int snp_map_cmd_buf_desc_list(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
726 {
727 int i;
728
729 snp_populate_cmd_buf_desc_list(cmd, cmd_buf, desc_list);
730
731 for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
732 struct cmd_buf_desc *desc = &desc_list[i];
733
734 if (!desc->paddr_ptr)
735 break;
736
737 if (snp_map_cmd_buf_desc(desc))
738 goto err_unmap;
739 }
740
741 return 0;
742
743 err_unmap:
744 for (i--; i >= 0; i--)
745 snp_unmap_cmd_buf_desc(&desc_list[i]);
746
747 return -EFAULT;
748 }
749
snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc * desc_list)750 static int snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc *desc_list)
751 {
752 int i, ret = 0;
753
754 for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
755 struct cmd_buf_desc *desc = &desc_list[i];
756
757 if (!desc->paddr_ptr)
758 break;
759
760 if (snp_unmap_cmd_buf_desc(&desc_list[i]))
761 ret = -EFAULT;
762 }
763
764 return ret;
765 }
766
sev_cmd_buf_writable(int cmd)767 static bool sev_cmd_buf_writable(int cmd)
768 {
769 switch (cmd) {
770 case SEV_CMD_PLATFORM_STATUS:
771 case SEV_CMD_GUEST_STATUS:
772 case SEV_CMD_LAUNCH_START:
773 case SEV_CMD_RECEIVE_START:
774 case SEV_CMD_LAUNCH_MEASURE:
775 case SEV_CMD_SEND_START:
776 case SEV_CMD_SEND_UPDATE_DATA:
777 case SEV_CMD_SEND_UPDATE_VMSA:
778 case SEV_CMD_PEK_CSR:
779 case SEV_CMD_PDH_CERT_EXPORT:
780 case SEV_CMD_GET_ID:
781 case SEV_CMD_ATTESTATION_REPORT:
782 return true;
783 default:
784 return false;
785 }
786 }
787
788 /* After SNP is INIT'ed, the behavior of legacy SEV commands is changed. */
snp_legacy_handling_needed(int cmd)789 static bool snp_legacy_handling_needed(int cmd)
790 {
791 struct sev_device *sev = psp_master->sev_data;
792
793 return cmd < SEV_CMD_SNP_INIT && sev->snp_initialized;
794 }
795
snp_prep_cmd_buf(int cmd,void * cmd_buf,struct cmd_buf_desc * desc_list)796 static int snp_prep_cmd_buf(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
797 {
798 if (!snp_legacy_handling_needed(cmd))
799 return 0;
800
801 if (snp_map_cmd_buf_desc_list(cmd, cmd_buf, desc_list))
802 return -EFAULT;
803
804 /*
805 * Before command execution, the command buffer needs to be put into
806 * the firmware-owned state.
807 */
808 if (sev_cmd_buf_writable(cmd)) {
809 if (rmp_mark_pages_firmware(__pa(cmd_buf), 1, true))
810 return -EFAULT;
811 }
812
813 return 0;
814 }
815
snp_reclaim_cmd_buf(int cmd,void * cmd_buf)816 static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf)
817 {
818 if (!snp_legacy_handling_needed(cmd))
819 return 0;
820
821 /*
822 * After command completion, the command buffer needs to be put back
823 * into the hypervisor-owned state.
824 */
825 if (sev_cmd_buf_writable(cmd))
826 if (snp_reclaim_pages(__pa(cmd_buf), 1, true))
827 return -EFAULT;
828
829 return 0;
830 }
831
__sev_do_cmd_locked(int cmd,void * data,int * psp_ret)832 static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
833 {
834 struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0};
835 struct psp_device *psp = psp_master;
836 struct sev_device *sev;
837 unsigned int cmdbuff_hi, cmdbuff_lo;
838 unsigned int phys_lsb, phys_msb;
839 unsigned int reg, ret = 0;
840 void *cmd_buf;
841 int buf_len;
842
843 if (!psp || !psp->sev_data)
844 return -ENODEV;
845
846 if (psp_dead)
847 return -EBUSY;
848
849 sev = psp->sev_data;
850
851 buf_len = sev_cmd_buffer_len(cmd);
852 if (WARN_ON_ONCE(!data != !buf_len))
853 return -EINVAL;
854
855 /*
856 * Copy the incoming data to driver's scratch buffer as __pa() will not
857 * work for some memory, e.g. vmalloc'd addresses, and @data may not be
858 * physically contiguous.
859 */
860 if (data) {
861 /*
862 * Commands are generally issued one at a time and require the
863 * sev_cmd_mutex, but there could be recursive firmware requests
864 * due to SEV_CMD_SNP_PAGE_RECLAIM needing to be issued while
865 * preparing buffers for another command. This is the only known
866 * case of nesting in the current code, so exactly one
867 * additional command buffer is available for that purpose.
868 */
869 if (!sev->cmd_buf_active) {
870 cmd_buf = sev->cmd_buf;
871 sev->cmd_buf_active = true;
872 } else if (!sev->cmd_buf_backup_active) {
873 cmd_buf = sev->cmd_buf_backup;
874 sev->cmd_buf_backup_active = true;
875 } else {
876 dev_err(sev->dev,
877 "SEV: too many firmware commands in progress, no command buffers available.\n");
878 return -EBUSY;
879 }
880
881 memcpy(cmd_buf, data, buf_len);
882
883 /*
884 * The behavior of the SEV-legacy commands is altered when the
885 * SNP firmware is in the INIT state.
886 */
887 ret = snp_prep_cmd_buf(cmd, cmd_buf, desc_list);
888 if (ret) {
889 dev_err(sev->dev,
890 "SEV: failed to prepare buffer for legacy command 0x%x. Error: %d\n",
891 cmd, ret);
892 return ret;
893 }
894 } else {
895 cmd_buf = sev->cmd_buf;
896 }
897
898 /* Get the physical address of the command buffer */
899 phys_lsb = data ? lower_32_bits(__psp_pa(cmd_buf)) : 0;
900 phys_msb = data ? upper_32_bits(__psp_pa(cmd_buf)) : 0;
901
902 dev_dbg(sev->dev, "sev command id %#x buffer 0x%08x%08x timeout %us\n",
903 cmd, phys_msb, phys_lsb, psp_timeout);
904
905 print_hex_dump_debug("(in): ", DUMP_PREFIX_OFFSET, 16, 2, data,
906 buf_len, false);
907
908 iowrite32(phys_lsb, sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
909 iowrite32(phys_msb, sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
910
911 sev->int_rcvd = 0;
912
913 reg = FIELD_PREP(SEV_CMDRESP_CMD, cmd);
914
915 /*
916 * If invoked during panic handling, local interrupts are disabled so
917 * the PSP command completion interrupt can't be used.
918 * sev_wait_cmd_ioc() already checks for interrupts disabled and
919 * polls for PSP command completion. Ensure we do not request an
920 * interrupt from the PSP if irqs disabled.
921 */
922 if (!irqs_disabled())
923 reg |= SEV_CMDRESP_IOC;
924
925 iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg);
926
927 /* wait for command completion */
928 ret = sev_wait_cmd_ioc(sev, ®, psp_timeout);
929 if (ret) {
930 if (psp_ret)
931 *psp_ret = 0;
932
933 dev_err(sev->dev, "sev command %#x timed out, disabling PSP\n", cmd);
934 psp_dead = true;
935
936 return ret;
937 }
938
939 psp_timeout = psp_cmd_timeout;
940
941 if (psp_ret)
942 *psp_ret = FIELD_GET(PSP_CMDRESP_STS, reg);
943
944 if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
945 dev_dbg(sev->dev, "sev command %#x failed (%#010lx)\n",
946 cmd, FIELD_GET(PSP_CMDRESP_STS, reg));
947
948 /*
949 * PSP firmware may report additional error information in the
950 * command buffer registers on error. Print contents of command
951 * buffer registers if they changed.
952 */
953 cmdbuff_hi = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
954 cmdbuff_lo = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
955 if (cmdbuff_hi != phys_msb || cmdbuff_lo != phys_lsb) {
956 dev_dbg(sev->dev, "Additional error information reported in cmdbuff:");
957 dev_dbg(sev->dev, " cmdbuff hi: %#010x\n", cmdbuff_hi);
958 dev_dbg(sev->dev, " cmdbuff lo: %#010x\n", cmdbuff_lo);
959 }
960 ret = -EIO;
961 } else {
962 ret = sev_write_init_ex_file_if_required(cmd);
963 }
964
965 /*
966 * Copy potential output from the PSP back to data. Do this even on
967 * failure in case the caller wants to glean something from the error.
968 */
969 if (data) {
970 int ret_reclaim;
971 /*
972 * Restore the page state after the command completes.
973 */
974 ret_reclaim = snp_reclaim_cmd_buf(cmd, cmd_buf);
975 if (ret_reclaim) {
976 dev_err(sev->dev,
977 "SEV: failed to reclaim buffer for legacy command %#x. Error: %d\n",
978 cmd, ret_reclaim);
979 return ret_reclaim;
980 }
981
982 memcpy(data, cmd_buf, buf_len);
983
984 if (sev->cmd_buf_backup_active)
985 sev->cmd_buf_backup_active = false;
986 else
987 sev->cmd_buf_active = false;
988
989 if (snp_unmap_cmd_buf_desc_list(desc_list))
990 return -EFAULT;
991 }
992
993 print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
994 buf_len, false);
995
996 return ret;
997 }
998
sev_do_cmd(int cmd,void * data,int * psp_ret)999 int sev_do_cmd(int cmd, void *data, int *psp_ret)
1000 {
1001 int rc;
1002
1003 mutex_lock(&sev_cmd_mutex);
1004 rc = __sev_do_cmd_locked(cmd, data, psp_ret);
1005 mutex_unlock(&sev_cmd_mutex);
1006
1007 return rc;
1008 }
1009 EXPORT_SYMBOL_GPL(sev_do_cmd);
1010
__sev_init_locked(int * error)1011 static int __sev_init_locked(int *error)
1012 {
1013 struct sev_data_init data;
1014
1015 memset(&data, 0, sizeof(data));
1016 if (sev_es_tmr) {
1017 /*
1018 * Do not include the encryption mask on the physical
1019 * address of the TMR (firmware should clear it anyway).
1020 */
1021 data.tmr_address = __pa(sev_es_tmr);
1022
1023 data.flags |= SEV_INIT_FLAGS_SEV_ES;
1024 data.tmr_len = sev_es_tmr_size;
1025 }
1026
1027 return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
1028 }
1029
__sev_init_ex_locked(int * error)1030 static int __sev_init_ex_locked(int *error)
1031 {
1032 struct sev_data_init_ex data;
1033
1034 memset(&data, 0, sizeof(data));
1035 data.length = sizeof(data);
1036 data.nv_address = __psp_pa(sev_init_ex_buffer);
1037 data.nv_len = NV_LENGTH;
1038
1039 if (sev_es_tmr) {
1040 /*
1041 * Do not include the encryption mask on the physical
1042 * address of the TMR (firmware should clear it anyway).
1043 */
1044 data.tmr_address = __pa(sev_es_tmr);
1045
1046 data.flags |= SEV_INIT_FLAGS_SEV_ES;
1047 data.tmr_len = sev_es_tmr_size;
1048 }
1049
1050 return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
1051 }
1052
__sev_do_init_locked(int * psp_ret)1053 static inline int __sev_do_init_locked(int *psp_ret)
1054 {
1055 if (sev_init_ex_buffer)
1056 return __sev_init_ex_locked(psp_ret);
1057 else
1058 return __sev_init_locked(psp_ret);
1059 }
1060
snp_set_hsave_pa(void * arg)1061 static void snp_set_hsave_pa(void *arg)
1062 {
1063 wrmsrl(MSR_VM_HSAVE_PA, 0);
1064 }
1065
snp_filter_reserved_mem_regions(struct resource * rs,void * arg)1066 static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg)
1067 {
1068 struct sev_data_range_list *range_list = arg;
1069 struct sev_data_range *range = &range_list->ranges[range_list->num_elements];
1070 size_t size;
1071
1072 /*
1073 * Ensure the list of HV_FIXED pages that will be passed to firmware
1074 * do not exceed the page-sized argument buffer.
1075 */
1076 if ((range_list->num_elements * sizeof(struct sev_data_range) +
1077 sizeof(struct sev_data_range_list)) > PAGE_SIZE)
1078 return -E2BIG;
1079
1080 switch (rs->desc) {
1081 case E820_TYPE_RESERVED:
1082 case E820_TYPE_PMEM:
1083 case E820_TYPE_ACPI:
1084 range->base = rs->start & PAGE_MASK;
1085 size = PAGE_ALIGN((rs->end + 1) - rs->start);
1086 range->page_count = size >> PAGE_SHIFT;
1087 range_list->num_elements++;
1088 break;
1089 default:
1090 break;
1091 }
1092
1093 return 0;
1094 }
1095
__sev_snp_init_locked(int * error)1096 static int __sev_snp_init_locked(int *error)
1097 {
1098 struct psp_device *psp = psp_master;
1099 struct sev_data_snp_init_ex data;
1100 struct sev_device *sev;
1101 void *arg = &data;
1102 int cmd, rc = 0;
1103
1104 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1105 return -ENODEV;
1106
1107 sev = psp->sev_data;
1108
1109 if (sev->snp_initialized)
1110 return 0;
1111
1112 if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) {
1113 dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n",
1114 SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR);
1115 return 0;
1116 }
1117
1118 /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */
1119 on_each_cpu(snp_set_hsave_pa, NULL, 1);
1120
1121 /*
1122 * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
1123 * of system physical address ranges to convert into HV-fixed page
1124 * states during the RMP initialization. For instance, the memory that
1125 * UEFI reserves should be included in the that list. This allows system
1126 * components that occasionally write to memory (e.g. logging to UEFI
1127 * reserved regions) to not fail due to RMP initialization and SNP
1128 * enablement.
1129 *
1130 */
1131 if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) {
1132 /*
1133 * Firmware checks that the pages containing the ranges enumerated
1134 * in the RANGES structure are either in the default page state or in the
1135 * firmware page state.
1136 */
1137 snp_range_list = kzalloc(PAGE_SIZE, GFP_KERNEL);
1138 if (!snp_range_list) {
1139 dev_err(sev->dev,
1140 "SEV: SNP_INIT_EX range list memory allocation failed\n");
1141 return -ENOMEM;
1142 }
1143
1144 /*
1145 * Retrieve all reserved memory regions from the e820 memory map
1146 * to be setup as HV-fixed pages.
1147 */
1148 rc = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM, 0, ~0,
1149 snp_range_list, snp_filter_reserved_mem_regions);
1150 if (rc) {
1151 dev_err(sev->dev,
1152 "SEV: SNP_INIT_EX walk_iomem_res_desc failed rc = %d\n", rc);
1153 return rc;
1154 }
1155
1156 memset(&data, 0, sizeof(data));
1157 data.init_rmp = 1;
1158 data.list_paddr_en = 1;
1159 data.list_paddr = __psp_pa(snp_range_list);
1160 cmd = SEV_CMD_SNP_INIT_EX;
1161 } else {
1162 cmd = SEV_CMD_SNP_INIT;
1163 arg = NULL;
1164 }
1165
1166 /*
1167 * The following sequence must be issued before launching the first SNP
1168 * guest to ensure all dirty cache lines are flushed, including from
1169 * updates to the RMP table itself via the RMPUPDATE instruction:
1170 *
1171 * - WBINVD on all running CPUs
1172 * - SEV_CMD_SNP_INIT[_EX] firmware command
1173 * - WBINVD on all running CPUs
1174 * - SEV_CMD_SNP_DF_FLUSH firmware command
1175 */
1176 wbinvd_on_all_cpus();
1177
1178 rc = __sev_do_cmd_locked(cmd, arg, error);
1179 if (rc)
1180 return rc;
1181
1182 /* Prepare for first SNP guest launch after INIT. */
1183 wbinvd_on_all_cpus();
1184 rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error);
1185 if (rc)
1186 return rc;
1187
1188 sev->snp_initialized = true;
1189 dev_dbg(sev->dev, "SEV-SNP firmware initialized\n");
1190
1191 sev_es_tmr_size = SNP_TMR_SIZE;
1192
1193 return rc;
1194 }
1195
__sev_platform_init_handle_tmr(struct sev_device * sev)1196 static void __sev_platform_init_handle_tmr(struct sev_device *sev)
1197 {
1198 if (sev_es_tmr)
1199 return;
1200
1201 /* Obtain the TMR memory area for SEV-ES use */
1202 sev_es_tmr = sev_fw_alloc(sev_es_tmr_size);
1203 if (sev_es_tmr) {
1204 /* Must flush the cache before giving it to the firmware */
1205 if (!sev->snp_initialized)
1206 clflush_cache_range(sev_es_tmr, sev_es_tmr_size);
1207 } else {
1208 dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n");
1209 }
1210 }
1211
1212 /*
1213 * If an init_ex_path is provided allocate a buffer for the file and
1214 * read in the contents. Additionally, if SNP is initialized, convert
1215 * the buffer pages to firmware pages.
1216 */
__sev_platform_init_handle_init_ex_path(struct sev_device * sev)1217 static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev)
1218 {
1219 struct page *page;
1220 int rc;
1221
1222 if (!init_ex_path)
1223 return 0;
1224
1225 if (sev_init_ex_buffer)
1226 return 0;
1227
1228 page = alloc_pages(GFP_KERNEL, get_order(NV_LENGTH));
1229 if (!page) {
1230 dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n");
1231 return -ENOMEM;
1232 }
1233
1234 sev_init_ex_buffer = page_address(page);
1235
1236 rc = sev_read_init_ex_file();
1237 if (rc)
1238 return rc;
1239
1240 /* If SEV-SNP is initialized, transition to firmware page. */
1241 if (sev->snp_initialized) {
1242 unsigned long npages;
1243
1244 npages = 1UL << get_order(NV_LENGTH);
1245 if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, false)) {
1246 dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n");
1247 return -ENOMEM;
1248 }
1249 }
1250
1251 return 0;
1252 }
1253
__sev_platform_init_locked(int * error)1254 static int __sev_platform_init_locked(int *error)
1255 {
1256 int rc, psp_ret = SEV_RET_NO_FW_CALL;
1257 struct sev_device *sev;
1258
1259 if (!psp_master || !psp_master->sev_data)
1260 return -ENODEV;
1261
1262 sev = psp_master->sev_data;
1263
1264 if (sev->state == SEV_STATE_INIT)
1265 return 0;
1266
1267 __sev_platform_init_handle_tmr(sev);
1268
1269 rc = __sev_platform_init_handle_init_ex_path(sev);
1270 if (rc)
1271 return rc;
1272
1273 rc = __sev_do_init_locked(&psp_ret);
1274 if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
1275 /*
1276 * Initialization command returned an integrity check failure
1277 * status code, meaning that firmware load and validation of SEV
1278 * related persistent data has failed. Retrying the
1279 * initialization function should succeed by replacing the state
1280 * with a reset state.
1281 */
1282 dev_err(sev->dev,
1283 "SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state.");
1284 rc = __sev_do_init_locked(&psp_ret);
1285 }
1286
1287 if (error)
1288 *error = psp_ret;
1289
1290 if (rc)
1291 return rc;
1292
1293 sev->state = SEV_STATE_INIT;
1294
1295 /* Prepare for first SEV guest launch after INIT */
1296 wbinvd_on_all_cpus();
1297 rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, error);
1298 if (rc)
1299 return rc;
1300
1301 dev_dbg(sev->dev, "SEV firmware initialized\n");
1302
1303 dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
1304 sev->api_minor, sev->build);
1305
1306 return 0;
1307 }
1308
_sev_platform_init_locked(struct sev_platform_init_args * args)1309 static int _sev_platform_init_locked(struct sev_platform_init_args *args)
1310 {
1311 struct sev_device *sev;
1312 int rc;
1313
1314 if (!psp_master || !psp_master->sev_data)
1315 return -ENODEV;
1316
1317 sev = psp_master->sev_data;
1318
1319 if (sev->state == SEV_STATE_INIT)
1320 return 0;
1321
1322 /*
1323 * Legacy guests cannot be running while SNP_INIT(_EX) is executing,
1324 * so perform SEV-SNP initialization at probe time.
1325 */
1326 rc = __sev_snp_init_locked(&args->error);
1327 if (rc && rc != -ENODEV) {
1328 /*
1329 * Don't abort the probe if SNP INIT failed,
1330 * continue to initialize the legacy SEV firmware.
1331 */
1332 dev_err(sev->dev, "SEV-SNP: failed to INIT rc %d, error %#x\n",
1333 rc, args->error);
1334 }
1335
1336 /* Defer legacy SEV/SEV-ES support if allowed by caller/module. */
1337 if (args->probe && !psp_init_on_probe)
1338 return 0;
1339
1340 return __sev_platform_init_locked(&args->error);
1341 }
1342
sev_platform_init(struct sev_platform_init_args * args)1343 int sev_platform_init(struct sev_platform_init_args *args)
1344 {
1345 int rc;
1346
1347 mutex_lock(&sev_cmd_mutex);
1348 rc = _sev_platform_init_locked(args);
1349 mutex_unlock(&sev_cmd_mutex);
1350
1351 return rc;
1352 }
1353 EXPORT_SYMBOL_GPL(sev_platform_init);
1354
__sev_platform_shutdown_locked(int * error)1355 static int __sev_platform_shutdown_locked(int *error)
1356 {
1357 struct psp_device *psp = psp_master;
1358 struct sev_device *sev;
1359 int ret;
1360
1361 if (!psp || !psp->sev_data)
1362 return 0;
1363
1364 sev = psp->sev_data;
1365
1366 if (sev->state == SEV_STATE_UNINIT)
1367 return 0;
1368
1369 ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error);
1370 if (ret)
1371 return ret;
1372
1373 sev->state = SEV_STATE_UNINIT;
1374 dev_dbg(sev->dev, "SEV firmware shutdown\n");
1375
1376 return ret;
1377 }
1378
sev_get_platform_state(int * state,int * error)1379 static int sev_get_platform_state(int *state, int *error)
1380 {
1381 struct sev_user_data_status data;
1382 int rc;
1383
1384 rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, error);
1385 if (rc)
1386 return rc;
1387
1388 *state = data.state;
1389 return rc;
1390 }
1391
sev_ioctl_do_reset(struct sev_issue_cmd * argp,bool writable)1392 static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable)
1393 {
1394 int state, rc;
1395
1396 if (!writable)
1397 return -EPERM;
1398
1399 /*
1400 * The SEV spec requires that FACTORY_RESET must be issued in
1401 * UNINIT state. Before we go further lets check if any guest is
1402 * active.
1403 *
1404 * If FW is in WORKING state then deny the request otherwise issue
1405 * SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET.
1406 *
1407 */
1408 rc = sev_get_platform_state(&state, &argp->error);
1409 if (rc)
1410 return rc;
1411
1412 if (state == SEV_STATE_WORKING)
1413 return -EBUSY;
1414
1415 if (state == SEV_STATE_INIT) {
1416 rc = __sev_platform_shutdown_locked(&argp->error);
1417 if (rc)
1418 return rc;
1419 }
1420
1421 return __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error);
1422 }
1423
sev_ioctl_do_platform_status(struct sev_issue_cmd * argp)1424 static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp)
1425 {
1426 struct sev_user_data_status data;
1427 int ret;
1428
1429 memset(&data, 0, sizeof(data));
1430
1431 ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error);
1432 if (ret)
1433 return ret;
1434
1435 if (copy_to_user((void __user *)argp->data, &data, sizeof(data)))
1436 ret = -EFAULT;
1437
1438 return ret;
1439 }
1440
sev_ioctl_do_pek_pdh_gen(int cmd,struct sev_issue_cmd * argp,bool writable)1441 static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable)
1442 {
1443 struct sev_device *sev = psp_master->sev_data;
1444 int rc;
1445
1446 if (!writable)
1447 return -EPERM;
1448
1449 if (sev->state == SEV_STATE_UNINIT) {
1450 rc = __sev_platform_init_locked(&argp->error);
1451 if (rc)
1452 return rc;
1453 }
1454
1455 return __sev_do_cmd_locked(cmd, NULL, &argp->error);
1456 }
1457
sev_ioctl_do_pek_csr(struct sev_issue_cmd * argp,bool writable)1458 static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable)
1459 {
1460 struct sev_device *sev = psp_master->sev_data;
1461 struct sev_user_data_pek_csr input;
1462 struct sev_data_pek_csr data;
1463 void __user *input_address;
1464 void *blob = NULL;
1465 int ret;
1466
1467 if (!writable)
1468 return -EPERM;
1469
1470 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
1471 return -EFAULT;
1472
1473 memset(&data, 0, sizeof(data));
1474
1475 /* userspace wants to query CSR length */
1476 if (!input.address || !input.length)
1477 goto cmd;
1478
1479 /* allocate a physically contiguous buffer to store the CSR blob */
1480 input_address = (void __user *)input.address;
1481 if (input.length > SEV_FW_BLOB_MAX_SIZE)
1482 return -EFAULT;
1483
1484 blob = kzalloc(input.length, GFP_KERNEL);
1485 if (!blob)
1486 return -ENOMEM;
1487
1488 data.address = __psp_pa(blob);
1489 data.len = input.length;
1490
1491 cmd:
1492 if (sev->state == SEV_STATE_UNINIT) {
1493 ret = __sev_platform_init_locked(&argp->error);
1494 if (ret)
1495 goto e_free_blob;
1496 }
1497
1498 ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error);
1499
1500 /* If we query the CSR length, FW responded with expected data. */
1501 input.length = data.len;
1502
1503 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
1504 ret = -EFAULT;
1505 goto e_free_blob;
1506 }
1507
1508 if (blob) {
1509 if (copy_to_user(input_address, blob, input.length))
1510 ret = -EFAULT;
1511 }
1512
1513 e_free_blob:
1514 kfree(blob);
1515 return ret;
1516 }
1517
psp_copy_user_blob(u64 uaddr,u32 len)1518 void *psp_copy_user_blob(u64 uaddr, u32 len)
1519 {
1520 if (!uaddr || !len)
1521 return ERR_PTR(-EINVAL);
1522
1523 /* verify that blob length does not exceed our limit */
1524 if (len > SEV_FW_BLOB_MAX_SIZE)
1525 return ERR_PTR(-EINVAL);
1526
1527 return memdup_user((void __user *)uaddr, len);
1528 }
1529 EXPORT_SYMBOL_GPL(psp_copy_user_blob);
1530
sev_get_api_version(void)1531 static int sev_get_api_version(void)
1532 {
1533 struct sev_device *sev = psp_master->sev_data;
1534 struct sev_user_data_status status;
1535 int error = 0, ret;
1536
1537 ret = sev_platform_status(&status, &error);
1538 if (ret) {
1539 dev_err(sev->dev,
1540 "SEV: failed to get status. Error: %#x\n", error);
1541 return 1;
1542 }
1543
1544 sev->api_major = status.api_major;
1545 sev->api_minor = status.api_minor;
1546 sev->build = status.build;
1547 sev->state = status.state;
1548
1549 return 0;
1550 }
1551
sev_get_firmware(struct device * dev,const struct firmware ** firmware)1552 static int sev_get_firmware(struct device *dev,
1553 const struct firmware **firmware)
1554 {
1555 char fw_name_specific[SEV_FW_NAME_SIZE];
1556 char fw_name_subset[SEV_FW_NAME_SIZE];
1557
1558 snprintf(fw_name_specific, sizeof(fw_name_specific),
1559 "amd/amd_sev_fam%.2xh_model%.2xh.sbin",
1560 boot_cpu_data.x86, boot_cpu_data.x86_model);
1561
1562 snprintf(fw_name_subset, sizeof(fw_name_subset),
1563 "amd/amd_sev_fam%.2xh_model%.1xxh.sbin",
1564 boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4);
1565
1566 /* Check for SEV FW for a particular model.
1567 * Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h
1568 *
1569 * or
1570 *
1571 * Check for SEV FW common to a subset of models.
1572 * Ex. amd_sev_fam17h_model0xh.sbin for
1573 * Family 17h Model 00h -- Family 17h Model 0Fh
1574 *
1575 * or
1576 *
1577 * Fall-back to using generic name: sev.fw
1578 */
1579 if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) ||
1580 (firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) ||
1581 (firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0))
1582 return 0;
1583
1584 return -ENOENT;
1585 }
1586
1587 /* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
sev_update_firmware(struct device * dev)1588 static int sev_update_firmware(struct device *dev)
1589 {
1590 struct sev_data_download_firmware *data;
1591 const struct firmware *firmware;
1592 int ret, error, order;
1593 struct page *p;
1594 u64 data_size;
1595
1596 if (!sev_version_greater_or_equal(0, 15)) {
1597 dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n");
1598 return -1;
1599 }
1600
1601 if (sev_get_firmware(dev, &firmware) == -ENOENT) {
1602 dev_dbg(dev, "No SEV firmware file present\n");
1603 return -1;
1604 }
1605
1606 /*
1607 * SEV FW expects the physical address given to it to be 32
1608 * byte aligned. Memory allocated has structure placed at the
1609 * beginning followed by the firmware being passed to the SEV
1610 * FW. Allocate enough memory for data structure + alignment
1611 * padding + SEV FW.
1612 */
1613 data_size = ALIGN(sizeof(struct sev_data_download_firmware), 32);
1614
1615 order = get_order(firmware->size + data_size);
1616 p = alloc_pages(GFP_KERNEL, order);
1617 if (!p) {
1618 ret = -1;
1619 goto fw_err;
1620 }
1621
1622 /*
1623 * Copy firmware data to a kernel allocated contiguous
1624 * memory region.
1625 */
1626 data = page_address(p);
1627 memcpy(page_address(p) + data_size, firmware->data, firmware->size);
1628
1629 data->address = __psp_pa(page_address(p) + data_size);
1630 data->len = firmware->size;
1631
1632 ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
1633
1634 /*
1635 * A quirk for fixing the committed TCB version, when upgrading from
1636 * earlier firmware version than 1.50.
1637 */
1638 if (!ret && !sev_version_greater_or_equal(1, 50))
1639 ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
1640
1641 if (ret)
1642 dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error);
1643
1644 __free_pages(p, order);
1645
1646 fw_err:
1647 release_firmware(firmware);
1648
1649 return ret;
1650 }
1651
__sev_snp_shutdown_locked(int * error,bool panic)1652 static int __sev_snp_shutdown_locked(int *error, bool panic)
1653 {
1654 struct psp_device *psp = psp_master;
1655 struct sev_device *sev;
1656 struct sev_data_snp_shutdown_ex data;
1657 int ret;
1658
1659 if (!psp || !psp->sev_data)
1660 return 0;
1661
1662 sev = psp->sev_data;
1663
1664 if (!sev->snp_initialized)
1665 return 0;
1666
1667 memset(&data, 0, sizeof(data));
1668 data.len = sizeof(data);
1669 data.iommu_snp_shutdown = 1;
1670
1671 /*
1672 * If invoked during panic handling, local interrupts are disabled
1673 * and all CPUs are stopped, so wbinvd_on_all_cpus() can't be called.
1674 * In that case, a wbinvd() is done on remote CPUs via the NMI
1675 * callback, so only a local wbinvd() is needed here.
1676 */
1677 if (!panic)
1678 wbinvd_on_all_cpus();
1679 else
1680 wbinvd();
1681
1682 ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error);
1683 /* SHUTDOWN may require DF_FLUSH */
1684 if (*error == SEV_RET_DFFLUSH_REQUIRED) {
1685 ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, NULL);
1686 if (ret) {
1687 dev_err(sev->dev, "SEV-SNP DF_FLUSH failed\n");
1688 return ret;
1689 }
1690 /* reissue the shutdown command */
1691 ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data,
1692 error);
1693 }
1694 if (ret) {
1695 dev_err(sev->dev, "SEV-SNP firmware shutdown failed\n");
1696 return ret;
1697 }
1698
1699 /*
1700 * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP
1701 * enforcement by the IOMMU and also transitions all pages
1702 * associated with the IOMMU to the Reclaim state.
1703 * Firmware was transitioning the IOMMU pages to Hypervisor state
1704 * before version 1.53. But, accounting for the number of assigned
1705 * 4kB pages in a 2M page was done incorrectly by not transitioning
1706 * to the Reclaim state. This resulted in RMP #PF when later accessing
1707 * the 2M page containing those pages during kexec boot. Hence, the
1708 * firmware now transitions these pages to Reclaim state and hypervisor
1709 * needs to transition these pages to shared state. SNP Firmware
1710 * version 1.53 and above are needed for kexec boot.
1711 */
1712 ret = amd_iommu_snp_disable();
1713 if (ret) {
1714 dev_err(sev->dev, "SNP IOMMU shutdown failed\n");
1715 return ret;
1716 }
1717
1718 sev->snp_initialized = false;
1719 dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n");
1720
1721 return ret;
1722 }
1723
sev_ioctl_do_pek_import(struct sev_issue_cmd * argp,bool writable)1724 static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable)
1725 {
1726 struct sev_device *sev = psp_master->sev_data;
1727 struct sev_user_data_pek_cert_import input;
1728 struct sev_data_pek_cert_import data;
1729 void *pek_blob, *oca_blob;
1730 int ret;
1731
1732 if (!writable)
1733 return -EPERM;
1734
1735 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
1736 return -EFAULT;
1737
1738 /* copy PEK certificate blobs from userspace */
1739 pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len);
1740 if (IS_ERR(pek_blob))
1741 return PTR_ERR(pek_blob);
1742
1743 data.reserved = 0;
1744 data.pek_cert_address = __psp_pa(pek_blob);
1745 data.pek_cert_len = input.pek_cert_len;
1746
1747 /* copy PEK certificate blobs from userspace */
1748 oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len);
1749 if (IS_ERR(oca_blob)) {
1750 ret = PTR_ERR(oca_blob);
1751 goto e_free_pek;
1752 }
1753
1754 data.oca_cert_address = __psp_pa(oca_blob);
1755 data.oca_cert_len = input.oca_cert_len;
1756
1757 /* If platform is not in INIT state then transition it to INIT */
1758 if (sev->state != SEV_STATE_INIT) {
1759 ret = __sev_platform_init_locked(&argp->error);
1760 if (ret)
1761 goto e_free_oca;
1762 }
1763
1764 ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error);
1765
1766 e_free_oca:
1767 kfree(oca_blob);
1768 e_free_pek:
1769 kfree(pek_blob);
1770 return ret;
1771 }
1772
sev_ioctl_do_get_id2(struct sev_issue_cmd * argp)1773 static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp)
1774 {
1775 struct sev_user_data_get_id2 input;
1776 struct sev_data_get_id data;
1777 void __user *input_address;
1778 void *id_blob = NULL;
1779 int ret;
1780
1781 /* SEV GET_ID is available from SEV API v0.16 and up */
1782 if (!sev_version_greater_or_equal(0, 16))
1783 return -ENOTSUPP;
1784
1785 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
1786 return -EFAULT;
1787
1788 input_address = (void __user *)input.address;
1789
1790 if (input.address && input.length) {
1791 /*
1792 * The length of the ID shouldn't be assumed by software since
1793 * it may change in the future. The allocation size is limited
1794 * to 1 << (PAGE_SHIFT + MAX_PAGE_ORDER) by the page allocator.
1795 * If the allocation fails, simply return ENOMEM rather than
1796 * warning in the kernel log.
1797 */
1798 id_blob = kzalloc(input.length, GFP_KERNEL | __GFP_NOWARN);
1799 if (!id_blob)
1800 return -ENOMEM;
1801
1802 data.address = __psp_pa(id_blob);
1803 data.len = input.length;
1804 } else {
1805 data.address = 0;
1806 data.len = 0;
1807 }
1808
1809 ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, &data, &argp->error);
1810
1811 /*
1812 * Firmware will return the length of the ID value (either the minimum
1813 * required length or the actual length written), return it to the user.
1814 */
1815 input.length = data.len;
1816
1817 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
1818 ret = -EFAULT;
1819 goto e_free;
1820 }
1821
1822 if (id_blob) {
1823 if (copy_to_user(input_address, id_blob, data.len)) {
1824 ret = -EFAULT;
1825 goto e_free;
1826 }
1827 }
1828
1829 e_free:
1830 kfree(id_blob);
1831
1832 return ret;
1833 }
1834
sev_ioctl_do_get_id(struct sev_issue_cmd * argp)1835 static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp)
1836 {
1837 struct sev_data_get_id *data;
1838 u64 data_size, user_size;
1839 void *id_blob, *mem;
1840 int ret;
1841
1842 /* SEV GET_ID available from SEV API v0.16 and up */
1843 if (!sev_version_greater_or_equal(0, 16))
1844 return -ENOTSUPP;
1845
1846 /* SEV FW expects the buffer it fills with the ID to be
1847 * 8-byte aligned. Memory allocated should be enough to
1848 * hold data structure + alignment padding + memory
1849 * where SEV FW writes the ID.
1850 */
1851 data_size = ALIGN(sizeof(struct sev_data_get_id), 8);
1852 user_size = sizeof(struct sev_user_data_get_id);
1853
1854 mem = kzalloc(data_size + user_size, GFP_KERNEL);
1855 if (!mem)
1856 return -ENOMEM;
1857
1858 data = mem;
1859 id_blob = mem + data_size;
1860
1861 data->address = __psp_pa(id_blob);
1862 data->len = user_size;
1863
1864 ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error);
1865 if (!ret) {
1866 if (copy_to_user((void __user *)argp->data, id_blob, data->len))
1867 ret = -EFAULT;
1868 }
1869
1870 kfree(mem);
1871
1872 return ret;
1873 }
1874
sev_ioctl_do_pdh_export(struct sev_issue_cmd * argp,bool writable)1875 static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
1876 {
1877 struct sev_device *sev = psp_master->sev_data;
1878 struct sev_user_data_pdh_cert_export input;
1879 void *pdh_blob = NULL, *cert_blob = NULL;
1880 struct sev_data_pdh_cert_export data;
1881 void __user *input_cert_chain_address;
1882 void __user *input_pdh_cert_address;
1883 int ret;
1884
1885 /* If platform is not in INIT state then transition it to INIT. */
1886 if (sev->state != SEV_STATE_INIT) {
1887 if (!writable)
1888 return -EPERM;
1889
1890 ret = __sev_platform_init_locked(&argp->error);
1891 if (ret)
1892 return ret;
1893 }
1894
1895 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
1896 return -EFAULT;
1897
1898 memset(&data, 0, sizeof(data));
1899
1900 /* Userspace wants to query the certificate length. */
1901 if (!input.pdh_cert_address ||
1902 !input.pdh_cert_len ||
1903 !input.cert_chain_address)
1904 goto cmd;
1905
1906 input_pdh_cert_address = (void __user *)input.pdh_cert_address;
1907 input_cert_chain_address = (void __user *)input.cert_chain_address;
1908
1909 /* Allocate a physically contiguous buffer to store the PDH blob. */
1910 if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE)
1911 return -EFAULT;
1912
1913 /* Allocate a physically contiguous buffer to store the cert chain blob. */
1914 if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE)
1915 return -EFAULT;
1916
1917 pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL);
1918 if (!pdh_blob)
1919 return -ENOMEM;
1920
1921 data.pdh_cert_address = __psp_pa(pdh_blob);
1922 data.pdh_cert_len = input.pdh_cert_len;
1923
1924 cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL);
1925 if (!cert_blob) {
1926 ret = -ENOMEM;
1927 goto e_free_pdh;
1928 }
1929
1930 data.cert_chain_address = __psp_pa(cert_blob);
1931 data.cert_chain_len = input.cert_chain_len;
1932
1933 cmd:
1934 ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error);
1935
1936 /* If we query the length, FW responded with expected data. */
1937 input.cert_chain_len = data.cert_chain_len;
1938 input.pdh_cert_len = data.pdh_cert_len;
1939
1940 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
1941 ret = -EFAULT;
1942 goto e_free_cert;
1943 }
1944
1945 if (pdh_blob) {
1946 if (copy_to_user(input_pdh_cert_address,
1947 pdh_blob, input.pdh_cert_len)) {
1948 ret = -EFAULT;
1949 goto e_free_cert;
1950 }
1951 }
1952
1953 if (cert_blob) {
1954 if (copy_to_user(input_cert_chain_address,
1955 cert_blob, input.cert_chain_len))
1956 ret = -EFAULT;
1957 }
1958
1959 e_free_cert:
1960 kfree(cert_blob);
1961 e_free_pdh:
1962 kfree(pdh_blob);
1963 return ret;
1964 }
1965
sev_ioctl_do_snp_platform_status(struct sev_issue_cmd * argp)1966 static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
1967 {
1968 struct sev_device *sev = psp_master->sev_data;
1969 struct sev_data_snp_addr buf;
1970 struct page *status_page;
1971 void *data;
1972 int ret;
1973
1974 if (!sev->snp_initialized || !argp->data)
1975 return -EINVAL;
1976
1977 status_page = alloc_page(GFP_KERNEL_ACCOUNT);
1978 if (!status_page)
1979 return -ENOMEM;
1980
1981 data = page_address(status_page);
1982
1983 /*
1984 * Firmware expects status page to be in firmware-owned state, otherwise
1985 * it will report firmware error code INVALID_PAGE_STATE (0x1A).
1986 */
1987 if (rmp_mark_pages_firmware(__pa(data), 1, true)) {
1988 ret = -EFAULT;
1989 goto cleanup;
1990 }
1991
1992 buf.address = __psp_pa(data);
1993 ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error);
1994
1995 /*
1996 * Status page will be transitioned to Reclaim state upon success, or
1997 * left in Firmware state in failure. Use snp_reclaim_pages() to
1998 * transition either case back to Hypervisor-owned state.
1999 */
2000 if (snp_reclaim_pages(__pa(data), 1, true))
2001 return -EFAULT;
2002
2003 if (ret)
2004 goto cleanup;
2005
2006 if (copy_to_user((void __user *)argp->data, data,
2007 sizeof(struct sev_user_data_snp_status)))
2008 ret = -EFAULT;
2009
2010 cleanup:
2011 __free_pages(status_page, 0);
2012 return ret;
2013 }
2014
sev_ioctl_do_snp_commit(struct sev_issue_cmd * argp)2015 static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp)
2016 {
2017 struct sev_device *sev = psp_master->sev_data;
2018 struct sev_data_snp_commit buf;
2019
2020 if (!sev->snp_initialized)
2021 return -EINVAL;
2022
2023 buf.len = sizeof(buf);
2024
2025 return __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error);
2026 }
2027
sev_ioctl_do_snp_set_config(struct sev_issue_cmd * argp,bool writable)2028 static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable)
2029 {
2030 struct sev_device *sev = psp_master->sev_data;
2031 struct sev_user_data_snp_config config;
2032
2033 if (!sev->snp_initialized || !argp->data)
2034 return -EINVAL;
2035
2036 if (!writable)
2037 return -EPERM;
2038
2039 if (copy_from_user(&config, (void __user *)argp->data, sizeof(config)))
2040 return -EFAULT;
2041
2042 return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error);
2043 }
2044
sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd * argp,bool writable)2045 static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
2046 {
2047 struct sev_device *sev = psp_master->sev_data;
2048 struct sev_user_data_snp_vlek_load input;
2049 void *blob;
2050 int ret;
2051
2052 if (!sev->snp_initialized || !argp->data)
2053 return -EINVAL;
2054
2055 if (!writable)
2056 return -EPERM;
2057
2058 if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input)))
2059 return -EFAULT;
2060
2061 if (input.len != sizeof(input) || input.vlek_wrapped_version != 0)
2062 return -EINVAL;
2063
2064 blob = psp_copy_user_blob(input.vlek_wrapped_address,
2065 sizeof(struct sev_user_data_snp_wrapped_vlek_hashstick));
2066 if (IS_ERR(blob))
2067 return PTR_ERR(blob);
2068
2069 input.vlek_wrapped_address = __psp_pa(blob);
2070
2071 ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error);
2072
2073 kfree(blob);
2074
2075 return ret;
2076 }
2077
sev_ioctl(struct file * file,unsigned int ioctl,unsigned long arg)2078 static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
2079 {
2080 void __user *argp = (void __user *)arg;
2081 struct sev_issue_cmd input;
2082 int ret = -EFAULT;
2083 bool writable = file->f_mode & FMODE_WRITE;
2084
2085 if (!psp_master || !psp_master->sev_data)
2086 return -ENODEV;
2087
2088 if (ioctl != SEV_ISSUE_CMD)
2089 return -EINVAL;
2090
2091 if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd)))
2092 return -EFAULT;
2093
2094 if (input.cmd > SEV_MAX)
2095 return -EINVAL;
2096
2097 mutex_lock(&sev_cmd_mutex);
2098
2099 switch (input.cmd) {
2100
2101 case SEV_FACTORY_RESET:
2102 ret = sev_ioctl_do_reset(&input, writable);
2103 break;
2104 case SEV_PLATFORM_STATUS:
2105 ret = sev_ioctl_do_platform_status(&input);
2106 break;
2107 case SEV_PEK_GEN:
2108 ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input, writable);
2109 break;
2110 case SEV_PDH_GEN:
2111 ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input, writable);
2112 break;
2113 case SEV_PEK_CSR:
2114 ret = sev_ioctl_do_pek_csr(&input, writable);
2115 break;
2116 case SEV_PEK_CERT_IMPORT:
2117 ret = sev_ioctl_do_pek_import(&input, writable);
2118 break;
2119 case SEV_PDH_CERT_EXPORT:
2120 ret = sev_ioctl_do_pdh_export(&input, writable);
2121 break;
2122 case SEV_GET_ID:
2123 pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n");
2124 ret = sev_ioctl_do_get_id(&input);
2125 break;
2126 case SEV_GET_ID2:
2127 ret = sev_ioctl_do_get_id2(&input);
2128 break;
2129 case SNP_PLATFORM_STATUS:
2130 ret = sev_ioctl_do_snp_platform_status(&input);
2131 break;
2132 case SNP_COMMIT:
2133 ret = sev_ioctl_do_snp_commit(&input);
2134 break;
2135 case SNP_SET_CONFIG:
2136 ret = sev_ioctl_do_snp_set_config(&input, writable);
2137 break;
2138 case SNP_VLEK_LOAD:
2139 ret = sev_ioctl_do_snp_vlek_load(&input, writable);
2140 break;
2141 default:
2142 ret = -EINVAL;
2143 goto out;
2144 }
2145
2146 if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd)))
2147 ret = -EFAULT;
2148 out:
2149 mutex_unlock(&sev_cmd_mutex);
2150
2151 return ret;
2152 }
2153
2154 static const struct file_operations sev_fops = {
2155 .owner = THIS_MODULE,
2156 .unlocked_ioctl = sev_ioctl,
2157 };
2158
sev_platform_status(struct sev_user_data_status * data,int * error)2159 int sev_platform_status(struct sev_user_data_status *data, int *error)
2160 {
2161 return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error);
2162 }
2163 EXPORT_SYMBOL_GPL(sev_platform_status);
2164
sev_guest_deactivate(struct sev_data_deactivate * data,int * error)2165 int sev_guest_deactivate(struct sev_data_deactivate *data, int *error)
2166 {
2167 return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error);
2168 }
2169 EXPORT_SYMBOL_GPL(sev_guest_deactivate);
2170
sev_guest_activate(struct sev_data_activate * data,int * error)2171 int sev_guest_activate(struct sev_data_activate *data, int *error)
2172 {
2173 return sev_do_cmd(SEV_CMD_ACTIVATE, data, error);
2174 }
2175 EXPORT_SYMBOL_GPL(sev_guest_activate);
2176
sev_guest_decommission(struct sev_data_decommission * data,int * error)2177 int sev_guest_decommission(struct sev_data_decommission *data, int *error)
2178 {
2179 return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error);
2180 }
2181 EXPORT_SYMBOL_GPL(sev_guest_decommission);
2182
sev_guest_df_flush(int * error)2183 int sev_guest_df_flush(int *error)
2184 {
2185 return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error);
2186 }
2187 EXPORT_SYMBOL_GPL(sev_guest_df_flush);
2188
sev_exit(struct kref * ref)2189 static void sev_exit(struct kref *ref)
2190 {
2191 misc_deregister(&misc_dev->misc);
2192 kfree(misc_dev);
2193 misc_dev = NULL;
2194 }
2195
sev_misc_init(struct sev_device * sev)2196 static int sev_misc_init(struct sev_device *sev)
2197 {
2198 struct device *dev = sev->dev;
2199 int ret;
2200
2201 /*
2202 * SEV feature support can be detected on multiple devices but the SEV
2203 * FW commands must be issued on the master. During probe, we do not
2204 * know the master hence we create /dev/sev on the first device probe.
2205 * sev_do_cmd() finds the right master device to which to issue the
2206 * command to the firmware.
2207 */
2208 if (!misc_dev) {
2209 struct miscdevice *misc;
2210
2211 misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL);
2212 if (!misc_dev)
2213 return -ENOMEM;
2214
2215 misc = &misc_dev->misc;
2216 misc->minor = MISC_DYNAMIC_MINOR;
2217 misc->name = DEVICE_NAME;
2218 misc->fops = &sev_fops;
2219
2220 ret = misc_register(misc);
2221 if (ret)
2222 return ret;
2223
2224 kref_init(&misc_dev->refcount);
2225 } else {
2226 kref_get(&misc_dev->refcount);
2227 }
2228
2229 init_waitqueue_head(&sev->int_queue);
2230 sev->misc = misc_dev;
2231 dev_dbg(dev, "registered SEV device\n");
2232
2233 return 0;
2234 }
2235
sev_dev_init(struct psp_device * psp)2236 int sev_dev_init(struct psp_device *psp)
2237 {
2238 struct device *dev = psp->dev;
2239 struct sev_device *sev;
2240 int ret = -ENOMEM;
2241
2242 if (!boot_cpu_has(X86_FEATURE_SEV)) {
2243 dev_info_once(dev, "SEV: memory encryption not enabled by BIOS\n");
2244 return 0;
2245 }
2246
2247 sev = devm_kzalloc(dev, sizeof(*sev), GFP_KERNEL);
2248 if (!sev)
2249 goto e_err;
2250
2251 sev->cmd_buf = (void *)devm_get_free_pages(dev, GFP_KERNEL, 1);
2252 if (!sev->cmd_buf)
2253 goto e_sev;
2254
2255 sev->cmd_buf_backup = (uint8_t *)sev->cmd_buf + PAGE_SIZE;
2256
2257 psp->sev_data = sev;
2258
2259 sev->dev = dev;
2260 sev->psp = psp;
2261
2262 sev->io_regs = psp->io_regs;
2263
2264 sev->vdata = (struct sev_vdata *)psp->vdata->sev;
2265 if (!sev->vdata) {
2266 ret = -ENODEV;
2267 dev_err(dev, "sev: missing driver data\n");
2268 goto e_buf;
2269 }
2270
2271 psp_set_sev_irq_handler(psp, sev_irq_handler, sev);
2272
2273 ret = sev_misc_init(sev);
2274 if (ret)
2275 goto e_irq;
2276
2277 dev_notice(dev, "sev enabled\n");
2278
2279 return 0;
2280
2281 e_irq:
2282 psp_clear_sev_irq_handler(psp);
2283 e_buf:
2284 devm_free_pages(dev, (unsigned long)sev->cmd_buf);
2285 e_sev:
2286 devm_kfree(dev, sev);
2287 e_err:
2288 psp->sev_data = NULL;
2289
2290 dev_notice(dev, "sev initialization failed\n");
2291
2292 return ret;
2293 }
2294
__sev_firmware_shutdown(struct sev_device * sev,bool panic)2295 static void __sev_firmware_shutdown(struct sev_device *sev, bool panic)
2296 {
2297 int error;
2298
2299 __sev_platform_shutdown_locked(NULL);
2300
2301 if (sev_es_tmr) {
2302 /*
2303 * The TMR area was encrypted, flush it from the cache.
2304 *
2305 * If invoked during panic handling, local interrupts are
2306 * disabled and all CPUs are stopped, so wbinvd_on_all_cpus()
2307 * can't be used. In that case, wbinvd() is done on remote CPUs
2308 * via the NMI callback, and done for this CPU later during
2309 * SNP shutdown, so wbinvd_on_all_cpus() can be skipped.
2310 */
2311 if (!panic)
2312 wbinvd_on_all_cpus();
2313
2314 __snp_free_firmware_pages(virt_to_page(sev_es_tmr),
2315 get_order(sev_es_tmr_size),
2316 true);
2317 sev_es_tmr = NULL;
2318 }
2319
2320 if (sev_init_ex_buffer) {
2321 __snp_free_firmware_pages(virt_to_page(sev_init_ex_buffer),
2322 get_order(NV_LENGTH),
2323 true);
2324 sev_init_ex_buffer = NULL;
2325 }
2326
2327 if (snp_range_list) {
2328 kfree(snp_range_list);
2329 snp_range_list = NULL;
2330 }
2331
2332 __sev_snp_shutdown_locked(&error, panic);
2333 }
2334
sev_firmware_shutdown(struct sev_device * sev)2335 static void sev_firmware_shutdown(struct sev_device *sev)
2336 {
2337 mutex_lock(&sev_cmd_mutex);
2338 __sev_firmware_shutdown(sev, false);
2339 mutex_unlock(&sev_cmd_mutex);
2340 }
2341
sev_dev_destroy(struct psp_device * psp)2342 void sev_dev_destroy(struct psp_device *psp)
2343 {
2344 struct sev_device *sev = psp->sev_data;
2345
2346 if (!sev)
2347 return;
2348
2349 sev_firmware_shutdown(sev);
2350
2351 if (sev->misc)
2352 kref_put(&misc_dev->refcount, sev_exit);
2353
2354 psp_clear_sev_irq_handler(psp);
2355 }
2356
snp_shutdown_on_panic(struct notifier_block * nb,unsigned long reason,void * arg)2357 static int snp_shutdown_on_panic(struct notifier_block *nb,
2358 unsigned long reason, void *arg)
2359 {
2360 struct sev_device *sev = psp_master->sev_data;
2361
2362 /*
2363 * If sev_cmd_mutex is already acquired, then it's likely
2364 * another PSP command is in flight and issuing a shutdown
2365 * would fail in unexpected ways. Rather than create even
2366 * more confusion during a panic, just bail out here.
2367 */
2368 if (mutex_is_locked(&sev_cmd_mutex))
2369 return NOTIFY_DONE;
2370
2371 __sev_firmware_shutdown(sev, true);
2372
2373 return NOTIFY_DONE;
2374 }
2375
2376 static struct notifier_block snp_panic_notifier = {
2377 .notifier_call = snp_shutdown_on_panic,
2378 };
2379
sev_issue_cmd_external_user(struct file * filep,unsigned int cmd,void * data,int * error)2380 int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd,
2381 void *data, int *error)
2382 {
2383 if (!filep || filep->f_op != &sev_fops)
2384 return -EBADF;
2385
2386 return sev_do_cmd(cmd, data, error);
2387 }
2388 EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
2389
sev_pci_init(void)2390 void sev_pci_init(void)
2391 {
2392 struct sev_device *sev = psp_master->sev_data;
2393 struct sev_platform_init_args args = {0};
2394 u8 api_major, api_minor, build;
2395 int rc;
2396
2397 if (!sev)
2398 return;
2399
2400 psp_timeout = psp_probe_timeout;
2401
2402 if (sev_get_api_version())
2403 goto err;
2404
2405 api_major = sev->api_major;
2406 api_minor = sev->api_minor;
2407 build = sev->build;
2408
2409 if (sev_update_firmware(sev->dev) == 0)
2410 sev_get_api_version();
2411
2412 if (api_major != sev->api_major || api_minor != sev->api_minor ||
2413 build != sev->build)
2414 dev_info(sev->dev, "SEV firmware updated from %d.%d.%d to %d.%d.%d\n",
2415 api_major, api_minor, build,
2416 sev->api_major, sev->api_minor, sev->build);
2417
2418 /* Initialize the platform */
2419 args.probe = true;
2420 rc = sev_platform_init(&args);
2421 if (rc)
2422 dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n",
2423 args.error, rc);
2424
2425 dev_info(sev->dev, "SEV%s API:%d.%d build:%d\n", sev->snp_initialized ?
2426 "-SNP" : "", sev->api_major, sev->api_minor, sev->build);
2427
2428 atomic_notifier_chain_register(&panic_notifier_list,
2429 &snp_panic_notifier);
2430 return;
2431
2432 err:
2433 sev_dev_destroy(psp_master);
2434
2435 psp_master->sev_data = NULL;
2436 }
2437
sev_pci_exit(void)2438 void sev_pci_exit(void)
2439 {
2440 struct sev_device *sev = psp_master->sev_data;
2441
2442 if (!sev)
2443 return;
2444
2445 sev_firmware_shutdown(sev);
2446
2447 atomic_notifier_chain_unregister(&panic_notifier_list,
2448 &snp_panic_notifier);
2449 }
2450