1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Secure Encrypted Virtualization (SEV) interface
4 *
5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
6 *
7 * Author: Brijesh Singh <brijesh.singh@amd.com>
8 */
9
10 #include <linux/bitfield.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/kthread.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/spinlock.h>
17 #include <linux/spinlock_types.h>
18 #include <linux/types.h>
19 #include <linux/mutex.h>
20 #include <linux/delay.h>
21 #include <linux/hw_random.h>
22 #include <linux/ccp.h>
23 #include <linux/firmware.h>
24 #include <linux/panic_notifier.h>
25 #include <linux/gfp.h>
26 #include <linux/cpufeature.h>
27 #include <linux/fs.h>
28 #include <linux/fs_struct.h>
29 #include <linux/psp.h>
30 #include <linux/amd-iommu.h>
31
32 #include <asm/smp.h>
33 #include <asm/cacheflush.h>
34 #include <asm/e820/types.h>
35 #include <asm/sev.h>
36 #include <asm/msr.h>
37
38 #include "psp-dev.h"
39 #include "sev-dev.h"
40
41 #define DEVICE_NAME "sev"
42 #define SEV_FW_FILE "amd/sev.fw"
43 #define SEV_FW_NAME_SIZE 64
44
45 /* Minimum firmware version required for the SEV-SNP support */
46 #define SNP_MIN_API_MAJOR 1
47 #define SNP_MIN_API_MINOR 51
48
49 /*
50 * Maximum number of firmware-writable buffers that might be specified
51 * in the parameters of a legacy SEV command buffer.
52 */
53 #define CMD_BUF_FW_WRITABLE_MAX 2
54
55 /* Leave room in the descriptor array for an end-of-list indicator. */
56 #define CMD_BUF_DESC_MAX (CMD_BUF_FW_WRITABLE_MAX + 1)
57
58 static DEFINE_MUTEX(sev_cmd_mutex);
59 static struct sev_misc_dev *misc_dev;
60
61 static int psp_cmd_timeout = 100;
62 module_param(psp_cmd_timeout, int, 0644);
63 MODULE_PARM_DESC(psp_cmd_timeout, " default timeout value, in seconds, for PSP commands");
64
65 static int psp_probe_timeout = 5;
66 module_param(psp_probe_timeout, int, 0644);
67 MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
68
69 static char *init_ex_path;
70 module_param(init_ex_path, charp, 0444);
71 MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
72
73 static bool psp_init_on_probe = true;
74 module_param(psp_init_on_probe, bool, 0444);
75 MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
76
77 MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
78 MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
79 MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
80 MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */
81
82 static bool psp_dead;
83 static int psp_timeout;
84
85 enum snp_hv_fixed_pages_state {
86 ALLOCATED,
87 HV_FIXED,
88 };
89
90 struct snp_hv_fixed_pages_entry {
91 struct list_head list;
92 struct page *page;
93 unsigned int order;
94 bool free;
95 enum snp_hv_fixed_pages_state page_state;
96 };
97
98 static LIST_HEAD(snp_hv_fixed_pages);
99
100 /* Trusted Memory Region (TMR):
101 * The TMR is a 1MB area that must be 1MB aligned. Use the page allocator
102 * to allocate the memory, which will return aligned memory for the specified
103 * allocation order.
104 *
105 * When SEV-SNP is enabled the TMR needs to be 2MB aligned and 2MB sized.
106 */
107 #define SEV_TMR_SIZE (1024 * 1024)
108 #define SNP_TMR_SIZE (2 * 1024 * 1024)
109
110 static void *sev_es_tmr;
111 static size_t sev_es_tmr_size = SEV_TMR_SIZE;
112
113 /* INIT_EX NV Storage:
114 * The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page
115 * allocator to allocate the memory, which will return aligned memory for the
116 * specified allocation order.
117 */
118 #define NV_LENGTH (32 * 1024)
119 static void *sev_init_ex_buffer;
120
121 /*
122 * SEV_DATA_RANGE_LIST:
123 * Array containing range of pages that firmware transitions to HV-fixed
124 * page state.
125 */
126 static struct sev_data_range_list *snp_range_list;
127
128 static void __sev_firmware_shutdown(struct sev_device *sev, bool panic);
129
130 static int snp_shutdown_on_panic(struct notifier_block *nb,
131 unsigned long reason, void *arg);
132
133 static struct notifier_block snp_panic_notifier = {
134 .notifier_call = snp_shutdown_on_panic,
135 };
136
sev_version_greater_or_equal(u8 maj,u8 min)137 static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
138 {
139 struct sev_device *sev = psp_master->sev_data;
140
141 if (sev->api_major > maj)
142 return true;
143
144 if (sev->api_major == maj && sev->api_minor >= min)
145 return true;
146
147 return false;
148 }
149
sev_irq_handler(int irq,void * data,unsigned int status)150 static void sev_irq_handler(int irq, void *data, unsigned int status)
151 {
152 struct sev_device *sev = data;
153 int reg;
154
155 /* Check if it is command completion: */
156 if (!(status & SEV_CMD_COMPLETE))
157 return;
158
159 /* Check if it is SEV command completion: */
160 reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
161 if (FIELD_GET(PSP_CMDRESP_RESP, reg)) {
162 sev->int_rcvd = 1;
163 wake_up(&sev->int_queue);
164 }
165 }
166
sev_wait_cmd_ioc(struct sev_device * sev,unsigned int * reg,unsigned int timeout)167 static int sev_wait_cmd_ioc(struct sev_device *sev,
168 unsigned int *reg, unsigned int timeout)
169 {
170 int ret;
171
172 /*
173 * If invoked during panic handling, local interrupts are disabled,
174 * so the PSP command completion interrupt can't be used. Poll for
175 * PSP command completion instead.
176 */
177 if (irqs_disabled()) {
178 unsigned long timeout_usecs = (timeout * USEC_PER_SEC) / 10;
179
180 /* Poll for SEV command completion: */
181 while (timeout_usecs--) {
182 *reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
183 if (*reg & PSP_CMDRESP_RESP)
184 return 0;
185
186 udelay(10);
187 }
188 return -ETIMEDOUT;
189 }
190
191 ret = wait_event_timeout(sev->int_queue,
192 sev->int_rcvd, timeout * HZ);
193 if (!ret)
194 return -ETIMEDOUT;
195
196 *reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
197
198 return 0;
199 }
200
sev_cmd_buffer_len(int cmd)201 static int sev_cmd_buffer_len(int cmd)
202 {
203 switch (cmd) {
204 case SEV_CMD_INIT: return sizeof(struct sev_data_init);
205 case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex);
206 case SEV_CMD_SNP_SHUTDOWN_EX: return sizeof(struct sev_data_snp_shutdown_ex);
207 case SEV_CMD_SNP_INIT_EX: return sizeof(struct sev_data_snp_init_ex);
208 case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status);
209 case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr);
210 case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import);
211 case SEV_CMD_PDH_CERT_EXPORT: return sizeof(struct sev_data_pdh_cert_export);
212 case SEV_CMD_LAUNCH_START: return sizeof(struct sev_data_launch_start);
213 case SEV_CMD_LAUNCH_UPDATE_DATA: return sizeof(struct sev_data_launch_update_data);
214 case SEV_CMD_LAUNCH_UPDATE_VMSA: return sizeof(struct sev_data_launch_update_vmsa);
215 case SEV_CMD_LAUNCH_FINISH: return sizeof(struct sev_data_launch_finish);
216 case SEV_CMD_LAUNCH_MEASURE: return sizeof(struct sev_data_launch_measure);
217 case SEV_CMD_ACTIVATE: return sizeof(struct sev_data_activate);
218 case SEV_CMD_DEACTIVATE: return sizeof(struct sev_data_deactivate);
219 case SEV_CMD_DECOMMISSION: return sizeof(struct sev_data_decommission);
220 case SEV_CMD_GUEST_STATUS: return sizeof(struct sev_data_guest_status);
221 case SEV_CMD_DBG_DECRYPT: return sizeof(struct sev_data_dbg);
222 case SEV_CMD_DBG_ENCRYPT: return sizeof(struct sev_data_dbg);
223 case SEV_CMD_SEND_START: return sizeof(struct sev_data_send_start);
224 case SEV_CMD_SEND_UPDATE_DATA: return sizeof(struct sev_data_send_update_data);
225 case SEV_CMD_SEND_UPDATE_VMSA: return sizeof(struct sev_data_send_update_vmsa);
226 case SEV_CMD_SEND_FINISH: return sizeof(struct sev_data_send_finish);
227 case SEV_CMD_RECEIVE_START: return sizeof(struct sev_data_receive_start);
228 case SEV_CMD_RECEIVE_FINISH: return sizeof(struct sev_data_receive_finish);
229 case SEV_CMD_RECEIVE_UPDATE_DATA: return sizeof(struct sev_data_receive_update_data);
230 case SEV_CMD_RECEIVE_UPDATE_VMSA: return sizeof(struct sev_data_receive_update_vmsa);
231 case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret);
232 case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware);
233 case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id);
234 case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report);
235 case SEV_CMD_SEND_CANCEL: return sizeof(struct sev_data_send_cancel);
236 case SEV_CMD_SNP_GCTX_CREATE: return sizeof(struct sev_data_snp_addr);
237 case SEV_CMD_SNP_LAUNCH_START: return sizeof(struct sev_data_snp_launch_start);
238 case SEV_CMD_SNP_LAUNCH_UPDATE: return sizeof(struct sev_data_snp_launch_update);
239 case SEV_CMD_SNP_ACTIVATE: return sizeof(struct sev_data_snp_activate);
240 case SEV_CMD_SNP_DECOMMISSION: return sizeof(struct sev_data_snp_addr);
241 case SEV_CMD_SNP_PAGE_RECLAIM: return sizeof(struct sev_data_snp_page_reclaim);
242 case SEV_CMD_SNP_GUEST_STATUS: return sizeof(struct sev_data_snp_guest_status);
243 case SEV_CMD_SNP_LAUNCH_FINISH: return sizeof(struct sev_data_snp_launch_finish);
244 case SEV_CMD_SNP_DBG_DECRYPT: return sizeof(struct sev_data_snp_dbg);
245 case SEV_CMD_SNP_DBG_ENCRYPT: return sizeof(struct sev_data_snp_dbg);
246 case SEV_CMD_SNP_PAGE_UNSMASH: return sizeof(struct sev_data_snp_page_unsmash);
247 case SEV_CMD_SNP_PLATFORM_STATUS: return sizeof(struct sev_data_snp_addr);
248 case SEV_CMD_SNP_GUEST_REQUEST: return sizeof(struct sev_data_snp_guest_request);
249 case SEV_CMD_SNP_CONFIG: return sizeof(struct sev_user_data_snp_config);
250 case SEV_CMD_SNP_COMMIT: return sizeof(struct sev_data_snp_commit);
251 default: return 0;
252 }
253
254 return 0;
255 }
256
open_file_as_root(const char * filename,int flags,umode_t mode)257 static struct file *open_file_as_root(const char *filename, int flags, umode_t mode)
258 {
259 struct file *fp;
260 struct path root;
261 struct cred *cred;
262 const struct cred *old_cred;
263
264 task_lock(&init_task);
265 get_fs_root(init_task.fs, &root);
266 task_unlock(&init_task);
267
268 cred = prepare_creds();
269 if (!cred)
270 return ERR_PTR(-ENOMEM);
271 cred->fsuid = GLOBAL_ROOT_UID;
272 old_cred = override_creds(cred);
273
274 fp = file_open_root(&root, filename, flags, mode);
275 path_put(&root);
276
277 put_cred(revert_creds(old_cred));
278
279 return fp;
280 }
281
sev_read_init_ex_file(void)282 static int sev_read_init_ex_file(void)
283 {
284 struct sev_device *sev = psp_master->sev_data;
285 struct file *fp;
286 ssize_t nread;
287
288 lockdep_assert_held(&sev_cmd_mutex);
289
290 if (!sev_init_ex_buffer)
291 return -EOPNOTSUPP;
292
293 fp = open_file_as_root(init_ex_path, O_RDONLY, 0);
294 if (IS_ERR(fp)) {
295 int ret = PTR_ERR(fp);
296
297 if (ret == -ENOENT) {
298 dev_info(sev->dev,
299 "SEV: %s does not exist and will be created later.\n",
300 init_ex_path);
301 ret = 0;
302 } else {
303 dev_err(sev->dev,
304 "SEV: could not open %s for read, error %d\n",
305 init_ex_path, ret);
306 }
307 return ret;
308 }
309
310 nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
311 if (nread != NV_LENGTH) {
312 dev_info(sev->dev,
313 "SEV: could not read %u bytes to non volatile memory area, ret %ld\n",
314 NV_LENGTH, nread);
315 }
316
317 dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
318 filp_close(fp, NULL);
319
320 return 0;
321 }
322
sev_write_init_ex_file(void)323 static int sev_write_init_ex_file(void)
324 {
325 struct sev_device *sev = psp_master->sev_data;
326 struct file *fp;
327 loff_t offset = 0;
328 ssize_t nwrite;
329
330 lockdep_assert_held(&sev_cmd_mutex);
331
332 if (!sev_init_ex_buffer)
333 return 0;
334
335 fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600);
336 if (IS_ERR(fp)) {
337 int ret = PTR_ERR(fp);
338
339 dev_err(sev->dev,
340 "SEV: could not open file for write, error %d\n",
341 ret);
342 return ret;
343 }
344
345 nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
346 vfs_fsync(fp, 0);
347 filp_close(fp, NULL);
348
349 if (nwrite != NV_LENGTH) {
350 dev_err(sev->dev,
351 "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
352 NV_LENGTH, nwrite);
353 return -EIO;
354 }
355
356 dev_dbg(sev->dev, "SEV: write successful to NV file\n");
357
358 return 0;
359 }
360
sev_write_init_ex_file_if_required(int cmd_id)361 static int sev_write_init_ex_file_if_required(int cmd_id)
362 {
363 lockdep_assert_held(&sev_cmd_mutex);
364
365 if (!sev_init_ex_buffer)
366 return 0;
367
368 /*
369 * Only a few platform commands modify the SPI/NV area, but none of the
370 * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
371 * PEK_CERT_IMPORT, and PDH_GEN do.
372 */
373 switch (cmd_id) {
374 case SEV_CMD_FACTORY_RESET:
375 case SEV_CMD_INIT_EX:
376 case SEV_CMD_PDH_GEN:
377 case SEV_CMD_PEK_CERT_IMPORT:
378 case SEV_CMD_PEK_GEN:
379 break;
380 default:
381 return 0;
382 }
383
384 return sev_write_init_ex_file();
385 }
386
387 /*
388 * snp_reclaim_pages() needs __sev_do_cmd_locked(), and __sev_do_cmd_locked()
389 * needs snp_reclaim_pages(), so a forward declaration is needed.
390 */
391 static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret);
392
snp_reclaim_pages(unsigned long paddr,unsigned int npages,bool locked)393 static int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked)
394 {
395 int ret, err, i;
396
397 paddr = __sme_clr(ALIGN_DOWN(paddr, PAGE_SIZE));
398
399 for (i = 0; i < npages; i++, paddr += PAGE_SIZE) {
400 struct sev_data_snp_page_reclaim data = {0};
401
402 data.paddr = paddr;
403
404 if (locked)
405 ret = __sev_do_cmd_locked(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
406 else
407 ret = sev_do_cmd(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
408
409 if (ret)
410 goto cleanup;
411
412 ret = rmp_make_shared(__phys_to_pfn(paddr), PG_LEVEL_4K);
413 if (ret)
414 goto cleanup;
415 }
416
417 return 0;
418
419 cleanup:
420 /*
421 * If there was a failure reclaiming the page then it is no longer safe
422 * to release it back to the system; leak it instead.
423 */
424 snp_leak_pages(__phys_to_pfn(paddr), npages - i);
425 return ret;
426 }
427
rmp_mark_pages_firmware(unsigned long paddr,unsigned int npages,bool locked)428 static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked)
429 {
430 unsigned long pfn = __sme_clr(paddr) >> PAGE_SHIFT;
431 int rc, i;
432
433 for (i = 0; i < npages; i++, pfn++) {
434 rc = rmp_make_private(pfn, 0, PG_LEVEL_4K, 0, true);
435 if (rc)
436 goto cleanup;
437 }
438
439 return 0;
440
441 cleanup:
442 /*
443 * Try unrolling the firmware state changes by
444 * reclaiming the pages which were already changed to the
445 * firmware state.
446 */
447 snp_reclaim_pages(paddr, i, locked);
448
449 return rc;
450 }
451
__snp_alloc_firmware_pages(gfp_t gfp_mask,int order,bool locked)452 static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order, bool locked)
453 {
454 unsigned long npages = 1ul << order, paddr;
455 struct sev_device *sev;
456 struct page *page;
457
458 if (!psp_master || !psp_master->sev_data)
459 return NULL;
460
461 page = alloc_pages(gfp_mask, order);
462 if (!page)
463 return NULL;
464
465 /* If SEV-SNP is initialized then add the page in RMP table. */
466 sev = psp_master->sev_data;
467 if (!sev->snp_initialized)
468 return page;
469
470 paddr = __pa((unsigned long)page_address(page));
471 if (rmp_mark_pages_firmware(paddr, npages, locked))
472 return NULL;
473
474 return page;
475 }
476
snp_alloc_firmware_page(gfp_t gfp_mask)477 void *snp_alloc_firmware_page(gfp_t gfp_mask)
478 {
479 struct page *page;
480
481 page = __snp_alloc_firmware_pages(gfp_mask, 0, false);
482
483 return page ? page_address(page) : NULL;
484 }
485 EXPORT_SYMBOL_GPL(snp_alloc_firmware_page);
486
__snp_free_firmware_pages(struct page * page,int order,bool locked)487 static void __snp_free_firmware_pages(struct page *page, int order, bool locked)
488 {
489 struct sev_device *sev = psp_master->sev_data;
490 unsigned long paddr, npages = 1ul << order;
491
492 if (!page)
493 return;
494
495 paddr = __pa((unsigned long)page_address(page));
496 if (sev->snp_initialized &&
497 snp_reclaim_pages(paddr, npages, locked))
498 return;
499
500 __free_pages(page, order);
501 }
502
snp_free_firmware_page(void * addr)503 void snp_free_firmware_page(void *addr)
504 {
505 if (!addr)
506 return;
507
508 __snp_free_firmware_pages(virt_to_page(addr), 0, false);
509 }
510 EXPORT_SYMBOL_GPL(snp_free_firmware_page);
511
sev_fw_alloc(unsigned long len)512 static void *sev_fw_alloc(unsigned long len)
513 {
514 struct page *page;
515
516 page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len), true);
517 if (!page)
518 return NULL;
519
520 return page_address(page);
521 }
522
523 /**
524 * struct cmd_buf_desc - descriptors for managing legacy SEV command address
525 * parameters corresponding to buffers that may be written to by firmware.
526 *
527 * @paddr_ptr: pointer to the address parameter in the command buffer which may
528 * need to be saved/restored depending on whether a bounce buffer
529 * is used. In the case of a bounce buffer, the command buffer
530 * needs to be updated with the address of the new bounce buffer
531 * snp_map_cmd_buf_desc() has allocated specifically for it. Must
532 * be NULL if this descriptor is only an end-of-list indicator.
533 *
534 * @paddr_orig: storage for the original address parameter, which can be used to
535 * restore the original value in @paddr_ptr in cases where it is
536 * replaced with the address of a bounce buffer.
537 *
538 * @len: length of buffer located at the address originally stored at @paddr_ptr
539 *
540 * @guest_owned: true if the address corresponds to guest-owned pages, in which
541 * case bounce buffers are not needed.
542 */
543 struct cmd_buf_desc {
544 u64 *paddr_ptr;
545 u64 paddr_orig;
546 u32 len;
547 bool guest_owned;
548 };
549
550 /*
551 * If a legacy SEV command parameter is a memory address, those pages in
552 * turn need to be transitioned to/from firmware-owned before/after
553 * executing the firmware command.
554 *
555 * Additionally, in cases where those pages are not guest-owned, a bounce
556 * buffer is needed in place of the original memory address parameter.
557 *
558 * A set of descriptors are used to keep track of this handling, and
559 * initialized here based on the specific commands being executed.
560 */
snp_populate_cmd_buf_desc_list(int cmd,void * cmd_buf,struct cmd_buf_desc * desc_list)561 static void snp_populate_cmd_buf_desc_list(int cmd, void *cmd_buf,
562 struct cmd_buf_desc *desc_list)
563 {
564 switch (cmd) {
565 case SEV_CMD_PDH_CERT_EXPORT: {
566 struct sev_data_pdh_cert_export *data = cmd_buf;
567
568 desc_list[0].paddr_ptr = &data->pdh_cert_address;
569 desc_list[0].len = data->pdh_cert_len;
570 desc_list[1].paddr_ptr = &data->cert_chain_address;
571 desc_list[1].len = data->cert_chain_len;
572 break;
573 }
574 case SEV_CMD_GET_ID: {
575 struct sev_data_get_id *data = cmd_buf;
576
577 desc_list[0].paddr_ptr = &data->address;
578 desc_list[0].len = data->len;
579 break;
580 }
581 case SEV_CMD_PEK_CSR: {
582 struct sev_data_pek_csr *data = cmd_buf;
583
584 desc_list[0].paddr_ptr = &data->address;
585 desc_list[0].len = data->len;
586 break;
587 }
588 case SEV_CMD_LAUNCH_UPDATE_DATA: {
589 struct sev_data_launch_update_data *data = cmd_buf;
590
591 desc_list[0].paddr_ptr = &data->address;
592 desc_list[0].len = data->len;
593 desc_list[0].guest_owned = true;
594 break;
595 }
596 case SEV_CMD_LAUNCH_UPDATE_VMSA: {
597 struct sev_data_launch_update_vmsa *data = cmd_buf;
598
599 desc_list[0].paddr_ptr = &data->address;
600 desc_list[0].len = data->len;
601 desc_list[0].guest_owned = true;
602 break;
603 }
604 case SEV_CMD_LAUNCH_MEASURE: {
605 struct sev_data_launch_measure *data = cmd_buf;
606
607 desc_list[0].paddr_ptr = &data->address;
608 desc_list[0].len = data->len;
609 break;
610 }
611 case SEV_CMD_LAUNCH_UPDATE_SECRET: {
612 struct sev_data_launch_secret *data = cmd_buf;
613
614 desc_list[0].paddr_ptr = &data->guest_address;
615 desc_list[0].len = data->guest_len;
616 desc_list[0].guest_owned = true;
617 break;
618 }
619 case SEV_CMD_DBG_DECRYPT: {
620 struct sev_data_dbg *data = cmd_buf;
621
622 desc_list[0].paddr_ptr = &data->dst_addr;
623 desc_list[0].len = data->len;
624 desc_list[0].guest_owned = true;
625 break;
626 }
627 case SEV_CMD_DBG_ENCRYPT: {
628 struct sev_data_dbg *data = cmd_buf;
629
630 desc_list[0].paddr_ptr = &data->dst_addr;
631 desc_list[0].len = data->len;
632 desc_list[0].guest_owned = true;
633 break;
634 }
635 case SEV_CMD_ATTESTATION_REPORT: {
636 struct sev_data_attestation_report *data = cmd_buf;
637
638 desc_list[0].paddr_ptr = &data->address;
639 desc_list[0].len = data->len;
640 break;
641 }
642 case SEV_CMD_SEND_START: {
643 struct sev_data_send_start *data = cmd_buf;
644
645 desc_list[0].paddr_ptr = &data->session_address;
646 desc_list[0].len = data->session_len;
647 break;
648 }
649 case SEV_CMD_SEND_UPDATE_DATA: {
650 struct sev_data_send_update_data *data = cmd_buf;
651
652 desc_list[0].paddr_ptr = &data->hdr_address;
653 desc_list[0].len = data->hdr_len;
654 desc_list[1].paddr_ptr = &data->trans_address;
655 desc_list[1].len = data->trans_len;
656 break;
657 }
658 case SEV_CMD_SEND_UPDATE_VMSA: {
659 struct sev_data_send_update_vmsa *data = cmd_buf;
660
661 desc_list[0].paddr_ptr = &data->hdr_address;
662 desc_list[0].len = data->hdr_len;
663 desc_list[1].paddr_ptr = &data->trans_address;
664 desc_list[1].len = data->trans_len;
665 break;
666 }
667 case SEV_CMD_RECEIVE_UPDATE_DATA: {
668 struct sev_data_receive_update_data *data = cmd_buf;
669
670 desc_list[0].paddr_ptr = &data->guest_address;
671 desc_list[0].len = data->guest_len;
672 desc_list[0].guest_owned = true;
673 break;
674 }
675 case SEV_CMD_RECEIVE_UPDATE_VMSA: {
676 struct sev_data_receive_update_vmsa *data = cmd_buf;
677
678 desc_list[0].paddr_ptr = &data->guest_address;
679 desc_list[0].len = data->guest_len;
680 desc_list[0].guest_owned = true;
681 break;
682 }
683 default:
684 break;
685 }
686 }
687
snp_map_cmd_buf_desc(struct cmd_buf_desc * desc)688 static int snp_map_cmd_buf_desc(struct cmd_buf_desc *desc)
689 {
690 unsigned int npages;
691
692 if (!desc->len)
693 return 0;
694
695 /* Allocate a bounce buffer if this isn't a guest owned page. */
696 if (!desc->guest_owned) {
697 struct page *page;
698
699 page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(desc->len));
700 if (!page) {
701 pr_warn("Failed to allocate bounce buffer for SEV legacy command.\n");
702 return -ENOMEM;
703 }
704
705 desc->paddr_orig = *desc->paddr_ptr;
706 *desc->paddr_ptr = __psp_pa(page_to_virt(page));
707 }
708
709 npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
710
711 /* Transition the buffer to firmware-owned. */
712 if (rmp_mark_pages_firmware(*desc->paddr_ptr, npages, true)) {
713 pr_warn("Error moving pages to firmware-owned state for SEV legacy command.\n");
714 return -EFAULT;
715 }
716
717 return 0;
718 }
719
snp_unmap_cmd_buf_desc(struct cmd_buf_desc * desc)720 static int snp_unmap_cmd_buf_desc(struct cmd_buf_desc *desc)
721 {
722 unsigned int npages;
723
724 if (!desc->len)
725 return 0;
726
727 npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
728
729 /* Transition the buffers back to hypervisor-owned. */
730 if (snp_reclaim_pages(*desc->paddr_ptr, npages, true)) {
731 pr_warn("Failed to reclaim firmware-owned pages while issuing SEV legacy command.\n");
732 return -EFAULT;
733 }
734
735 /* Copy data from bounce buffer and then free it. */
736 if (!desc->guest_owned) {
737 void *bounce_buf = __va(__sme_clr(*desc->paddr_ptr));
738 void *dst_buf = __va(__sme_clr(desc->paddr_orig));
739
740 memcpy(dst_buf, bounce_buf, desc->len);
741 __free_pages(virt_to_page(bounce_buf), get_order(desc->len));
742
743 /* Restore the original address in the command buffer. */
744 *desc->paddr_ptr = desc->paddr_orig;
745 }
746
747 return 0;
748 }
749
snp_map_cmd_buf_desc_list(int cmd,void * cmd_buf,struct cmd_buf_desc * desc_list)750 static int snp_map_cmd_buf_desc_list(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
751 {
752 int i;
753
754 snp_populate_cmd_buf_desc_list(cmd, cmd_buf, desc_list);
755
756 for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
757 struct cmd_buf_desc *desc = &desc_list[i];
758
759 if (!desc->paddr_ptr)
760 break;
761
762 if (snp_map_cmd_buf_desc(desc))
763 goto err_unmap;
764 }
765
766 return 0;
767
768 err_unmap:
769 for (i--; i >= 0; i--)
770 snp_unmap_cmd_buf_desc(&desc_list[i]);
771
772 return -EFAULT;
773 }
774
snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc * desc_list)775 static int snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc *desc_list)
776 {
777 int i, ret = 0;
778
779 for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
780 struct cmd_buf_desc *desc = &desc_list[i];
781
782 if (!desc->paddr_ptr)
783 break;
784
785 if (snp_unmap_cmd_buf_desc(&desc_list[i]))
786 ret = -EFAULT;
787 }
788
789 return ret;
790 }
791
sev_cmd_buf_writable(int cmd)792 static bool sev_cmd_buf_writable(int cmd)
793 {
794 switch (cmd) {
795 case SEV_CMD_PLATFORM_STATUS:
796 case SEV_CMD_GUEST_STATUS:
797 case SEV_CMD_LAUNCH_START:
798 case SEV_CMD_RECEIVE_START:
799 case SEV_CMD_LAUNCH_MEASURE:
800 case SEV_CMD_SEND_START:
801 case SEV_CMD_SEND_UPDATE_DATA:
802 case SEV_CMD_SEND_UPDATE_VMSA:
803 case SEV_CMD_PEK_CSR:
804 case SEV_CMD_PDH_CERT_EXPORT:
805 case SEV_CMD_GET_ID:
806 case SEV_CMD_ATTESTATION_REPORT:
807 return true;
808 default:
809 return false;
810 }
811 }
812
813 /* After SNP is INIT'ed, the behavior of legacy SEV commands is changed. */
snp_legacy_handling_needed(int cmd)814 static bool snp_legacy_handling_needed(int cmd)
815 {
816 struct sev_device *sev = psp_master->sev_data;
817
818 return cmd < SEV_CMD_SNP_INIT && sev->snp_initialized;
819 }
820
snp_prep_cmd_buf(int cmd,void * cmd_buf,struct cmd_buf_desc * desc_list)821 static int snp_prep_cmd_buf(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
822 {
823 if (!snp_legacy_handling_needed(cmd))
824 return 0;
825
826 if (snp_map_cmd_buf_desc_list(cmd, cmd_buf, desc_list))
827 return -EFAULT;
828
829 /*
830 * Before command execution, the command buffer needs to be put into
831 * the firmware-owned state.
832 */
833 if (sev_cmd_buf_writable(cmd)) {
834 if (rmp_mark_pages_firmware(__pa(cmd_buf), 1, true))
835 return -EFAULT;
836 }
837
838 return 0;
839 }
840
snp_reclaim_cmd_buf(int cmd,void * cmd_buf)841 static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf)
842 {
843 if (!snp_legacy_handling_needed(cmd))
844 return 0;
845
846 /*
847 * After command completion, the command buffer needs to be put back
848 * into the hypervisor-owned state.
849 */
850 if (sev_cmd_buf_writable(cmd))
851 if (snp_reclaim_pages(__pa(cmd_buf), 1, true))
852 return -EFAULT;
853
854 return 0;
855 }
856
__sev_do_cmd_locked(int cmd,void * data,int * psp_ret)857 static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
858 {
859 struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0};
860 struct psp_device *psp = psp_master;
861 struct sev_device *sev;
862 unsigned int cmdbuff_hi, cmdbuff_lo;
863 unsigned int phys_lsb, phys_msb;
864 unsigned int reg, ret = 0;
865 void *cmd_buf;
866 int buf_len;
867
868 if (!psp || !psp->sev_data)
869 return -ENODEV;
870
871 if (psp_dead)
872 return -EBUSY;
873
874 sev = psp->sev_data;
875
876 buf_len = sev_cmd_buffer_len(cmd);
877 if (WARN_ON_ONCE(!data != !buf_len))
878 return -EINVAL;
879
880 /*
881 * Copy the incoming data to driver's scratch buffer as __pa() will not
882 * work for some memory, e.g. vmalloc'd addresses, and @data may not be
883 * physically contiguous.
884 */
885 if (data) {
886 /*
887 * Commands are generally issued one at a time and require the
888 * sev_cmd_mutex, but there could be recursive firmware requests
889 * due to SEV_CMD_SNP_PAGE_RECLAIM needing to be issued while
890 * preparing buffers for another command. This is the only known
891 * case of nesting in the current code, so exactly one
892 * additional command buffer is available for that purpose.
893 */
894 if (!sev->cmd_buf_active) {
895 cmd_buf = sev->cmd_buf;
896 sev->cmd_buf_active = true;
897 } else if (!sev->cmd_buf_backup_active) {
898 cmd_buf = sev->cmd_buf_backup;
899 sev->cmd_buf_backup_active = true;
900 } else {
901 dev_err(sev->dev,
902 "SEV: too many firmware commands in progress, no command buffers available.\n");
903 return -EBUSY;
904 }
905
906 memcpy(cmd_buf, data, buf_len);
907
908 /*
909 * The behavior of the SEV-legacy commands is altered when the
910 * SNP firmware is in the INIT state.
911 */
912 ret = snp_prep_cmd_buf(cmd, cmd_buf, desc_list);
913 if (ret) {
914 dev_err(sev->dev,
915 "SEV: failed to prepare buffer for legacy command 0x%x. Error: %d\n",
916 cmd, ret);
917 return ret;
918 }
919 } else {
920 cmd_buf = sev->cmd_buf;
921 }
922
923 /* Get the physical address of the command buffer */
924 phys_lsb = data ? lower_32_bits(__psp_pa(cmd_buf)) : 0;
925 phys_msb = data ? upper_32_bits(__psp_pa(cmd_buf)) : 0;
926
927 dev_dbg(sev->dev, "sev command id %#x buffer 0x%08x%08x timeout %us\n",
928 cmd, phys_msb, phys_lsb, psp_timeout);
929
930 print_hex_dump_debug("(in): ", DUMP_PREFIX_OFFSET, 16, 2, data,
931 buf_len, false);
932
933 iowrite32(phys_lsb, sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
934 iowrite32(phys_msb, sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
935
936 sev->int_rcvd = 0;
937
938 reg = FIELD_PREP(SEV_CMDRESP_CMD, cmd);
939
940 /*
941 * If invoked during panic handling, local interrupts are disabled so
942 * the PSP command completion interrupt can't be used.
943 * sev_wait_cmd_ioc() already checks for interrupts disabled and
944 * polls for PSP command completion. Ensure we do not request an
945 * interrupt from the PSP if irqs disabled.
946 */
947 if (!irqs_disabled())
948 reg |= SEV_CMDRESP_IOC;
949
950 iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg);
951
952 /* wait for command completion */
953 ret = sev_wait_cmd_ioc(sev, ®, psp_timeout);
954 if (ret) {
955 if (psp_ret)
956 *psp_ret = 0;
957
958 dev_err(sev->dev, "sev command %#x timed out, disabling PSP\n", cmd);
959 psp_dead = true;
960
961 return ret;
962 }
963
964 psp_timeout = psp_cmd_timeout;
965
966 if (psp_ret)
967 *psp_ret = FIELD_GET(PSP_CMDRESP_STS, reg);
968
969 if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
970 dev_dbg(sev->dev, "sev command %#x failed (%#010lx)\n",
971 cmd, FIELD_GET(PSP_CMDRESP_STS, reg));
972
973 /*
974 * PSP firmware may report additional error information in the
975 * command buffer registers on error. Print contents of command
976 * buffer registers if they changed.
977 */
978 cmdbuff_hi = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
979 cmdbuff_lo = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
980 if (cmdbuff_hi != phys_msb || cmdbuff_lo != phys_lsb) {
981 dev_dbg(sev->dev, "Additional error information reported in cmdbuff:");
982 dev_dbg(sev->dev, " cmdbuff hi: %#010x\n", cmdbuff_hi);
983 dev_dbg(sev->dev, " cmdbuff lo: %#010x\n", cmdbuff_lo);
984 }
985 ret = -EIO;
986 } else {
987 ret = sev_write_init_ex_file_if_required(cmd);
988 }
989
990 /*
991 * Copy potential output from the PSP back to data. Do this even on
992 * failure in case the caller wants to glean something from the error.
993 */
994 if (data) {
995 int ret_reclaim;
996 /*
997 * Restore the page state after the command completes.
998 */
999 ret_reclaim = snp_reclaim_cmd_buf(cmd, cmd_buf);
1000 if (ret_reclaim) {
1001 dev_err(sev->dev,
1002 "SEV: failed to reclaim buffer for legacy command %#x. Error: %d\n",
1003 cmd, ret_reclaim);
1004 return ret_reclaim;
1005 }
1006
1007 memcpy(data, cmd_buf, buf_len);
1008
1009 if (sev->cmd_buf_backup_active)
1010 sev->cmd_buf_backup_active = false;
1011 else
1012 sev->cmd_buf_active = false;
1013
1014 if (snp_unmap_cmd_buf_desc_list(desc_list))
1015 return -EFAULT;
1016 }
1017
1018 print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
1019 buf_len, false);
1020
1021 return ret;
1022 }
1023
sev_do_cmd(int cmd,void * data,int * psp_ret)1024 int sev_do_cmd(int cmd, void *data, int *psp_ret)
1025 {
1026 int rc;
1027
1028 mutex_lock(&sev_cmd_mutex);
1029 rc = __sev_do_cmd_locked(cmd, data, psp_ret);
1030 mutex_unlock(&sev_cmd_mutex);
1031
1032 return rc;
1033 }
1034 EXPORT_SYMBOL_GPL(sev_do_cmd);
1035
__sev_init_locked(int * error)1036 static int __sev_init_locked(int *error)
1037 {
1038 struct sev_data_init data;
1039
1040 memset(&data, 0, sizeof(data));
1041 if (sev_es_tmr) {
1042 /*
1043 * Do not include the encryption mask on the physical
1044 * address of the TMR (firmware should clear it anyway).
1045 */
1046 data.tmr_address = __pa(sev_es_tmr);
1047
1048 data.flags |= SEV_INIT_FLAGS_SEV_ES;
1049 data.tmr_len = sev_es_tmr_size;
1050 }
1051
1052 return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
1053 }
1054
__sev_init_ex_locked(int * error)1055 static int __sev_init_ex_locked(int *error)
1056 {
1057 struct sev_data_init_ex data;
1058
1059 memset(&data, 0, sizeof(data));
1060 data.length = sizeof(data);
1061 data.nv_address = __psp_pa(sev_init_ex_buffer);
1062 data.nv_len = NV_LENGTH;
1063
1064 if (sev_es_tmr) {
1065 /*
1066 * Do not include the encryption mask on the physical
1067 * address of the TMR (firmware should clear it anyway).
1068 */
1069 data.tmr_address = __pa(sev_es_tmr);
1070
1071 data.flags |= SEV_INIT_FLAGS_SEV_ES;
1072 data.tmr_len = sev_es_tmr_size;
1073 }
1074
1075 return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
1076 }
1077
__sev_do_init_locked(int * psp_ret)1078 static inline int __sev_do_init_locked(int *psp_ret)
1079 {
1080 if (sev_init_ex_buffer)
1081 return __sev_init_ex_locked(psp_ret);
1082 else
1083 return __sev_init_locked(psp_ret);
1084 }
1085
snp_set_hsave_pa(void * arg)1086 static void snp_set_hsave_pa(void *arg)
1087 {
1088 wrmsrq(MSR_VM_HSAVE_PA, 0);
1089 }
1090
1091 /* Hypervisor Fixed pages API interface */
snp_hv_fixed_pages_state_update(struct sev_device * sev,enum snp_hv_fixed_pages_state page_state)1092 static void snp_hv_fixed_pages_state_update(struct sev_device *sev,
1093 enum snp_hv_fixed_pages_state page_state)
1094 {
1095 struct snp_hv_fixed_pages_entry *entry;
1096
1097 /* List is protected by sev_cmd_mutex */
1098 lockdep_assert_held(&sev_cmd_mutex);
1099
1100 if (list_empty(&snp_hv_fixed_pages))
1101 return;
1102
1103 list_for_each_entry(entry, &snp_hv_fixed_pages, list)
1104 entry->page_state = page_state;
1105 }
1106
1107 /*
1108 * Allocate HV_FIXED pages in 2MB aligned sizes to ensure the whole
1109 * 2MB pages are marked as HV_FIXED.
1110 */
snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages)1111 struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages)
1112 {
1113 struct psp_device *psp_master = psp_get_master_device();
1114 struct snp_hv_fixed_pages_entry *entry;
1115 struct sev_device *sev;
1116 unsigned int order;
1117 struct page *page;
1118
1119 if (!psp_master || !psp_master->sev_data)
1120 return NULL;
1121
1122 sev = psp_master->sev_data;
1123
1124 order = get_order(PMD_SIZE * num_2mb_pages);
1125
1126 /*
1127 * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1128 * also needs to be protected using the same mutex.
1129 */
1130 guard(mutex)(&sev_cmd_mutex);
1131
1132 /*
1133 * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed
1134 * page state, fail if SNP is already initialized.
1135 */
1136 if (sev->snp_initialized)
1137 return NULL;
1138
1139 /* Re-use freed pages that match the request */
1140 list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1141 /* Hypervisor fixed page allocator implements exact fit policy */
1142 if (entry->order == order && entry->free) {
1143 entry->free = false;
1144 memset(page_address(entry->page), 0,
1145 (1 << entry->order) * PAGE_SIZE);
1146 return entry->page;
1147 }
1148 }
1149
1150 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
1151 if (!page)
1152 return NULL;
1153
1154 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1155 if (!entry) {
1156 __free_pages(page, order);
1157 return NULL;
1158 }
1159
1160 entry->page = page;
1161 entry->order = order;
1162 list_add_tail(&entry->list, &snp_hv_fixed_pages);
1163
1164 return page;
1165 }
1166
snp_free_hv_fixed_pages(struct page * page)1167 void snp_free_hv_fixed_pages(struct page *page)
1168 {
1169 struct psp_device *psp_master = psp_get_master_device();
1170 struct snp_hv_fixed_pages_entry *entry, *nentry;
1171
1172 if (!psp_master || !psp_master->sev_data)
1173 return;
1174
1175 /*
1176 * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1177 * also needs to be protected using the same mutex.
1178 */
1179 guard(mutex)(&sev_cmd_mutex);
1180
1181 list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) {
1182 if (entry->page != page)
1183 continue;
1184
1185 /*
1186 * HV_FIXED page state cannot be changed until reboot
1187 * and they cannot be used by an SNP guest, so they cannot
1188 * be returned back to the page allocator.
1189 * Mark the pages as free internally to allow possible re-use.
1190 */
1191 if (entry->page_state == HV_FIXED) {
1192 entry->free = true;
1193 } else {
1194 __free_pages(page, entry->order);
1195 list_del(&entry->list);
1196 kfree(entry);
1197 }
1198 return;
1199 }
1200 }
1201
snp_add_hv_fixed_pages(struct sev_device * sev,struct sev_data_range_list * range_list)1202 static void snp_add_hv_fixed_pages(struct sev_device *sev, struct sev_data_range_list *range_list)
1203 {
1204 struct snp_hv_fixed_pages_entry *entry;
1205 struct sev_data_range *range;
1206 int num_elements;
1207
1208 lockdep_assert_held(&sev_cmd_mutex);
1209
1210 if (list_empty(&snp_hv_fixed_pages))
1211 return;
1212
1213 num_elements = list_count_nodes(&snp_hv_fixed_pages) +
1214 range_list->num_elements;
1215
1216 /*
1217 * Ensure the list of HV_FIXED pages that will be passed to firmware
1218 * do not exceed the page-sized argument buffer.
1219 */
1220 if (num_elements * sizeof(*range) + sizeof(*range_list) > PAGE_SIZE) {
1221 dev_warn(sev->dev, "Additional HV_Fixed pages cannot be accommodated, omitting\n");
1222 return;
1223 }
1224
1225 range = &range_list->ranges[range_list->num_elements];
1226 list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1227 range->base = page_to_pfn(entry->page) << PAGE_SHIFT;
1228 range->page_count = 1 << entry->order;
1229 range++;
1230 }
1231 range_list->num_elements = num_elements;
1232 }
1233
snp_leak_hv_fixed_pages(void)1234 static void snp_leak_hv_fixed_pages(void)
1235 {
1236 struct snp_hv_fixed_pages_entry *entry;
1237
1238 /* List is protected by sev_cmd_mutex */
1239 lockdep_assert_held(&sev_cmd_mutex);
1240
1241 if (list_empty(&snp_hv_fixed_pages))
1242 return;
1243
1244 list_for_each_entry(entry, &snp_hv_fixed_pages, list)
1245 if (entry->page_state == HV_FIXED)
1246 __snp_leak_pages(page_to_pfn(entry->page),
1247 1 << entry->order, false);
1248 }
1249
snp_filter_reserved_mem_regions(struct resource * rs,void * arg)1250 static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg)
1251 {
1252 struct sev_data_range_list *range_list = arg;
1253 struct sev_data_range *range = &range_list->ranges[range_list->num_elements];
1254 size_t size;
1255
1256 /*
1257 * Ensure the list of HV_FIXED pages that will be passed to firmware
1258 * do not exceed the page-sized argument buffer.
1259 */
1260 if ((range_list->num_elements * sizeof(struct sev_data_range) +
1261 sizeof(struct sev_data_range_list)) > PAGE_SIZE)
1262 return -E2BIG;
1263
1264 switch (rs->desc) {
1265 case E820_TYPE_RESERVED:
1266 case E820_TYPE_PMEM:
1267 case E820_TYPE_ACPI:
1268 range->base = rs->start & PAGE_MASK;
1269 size = PAGE_ALIGN((rs->end + 1) - rs->start);
1270 range->page_count = size >> PAGE_SHIFT;
1271 range_list->num_elements++;
1272 break;
1273 default:
1274 break;
1275 }
1276
1277 return 0;
1278 }
1279
__sev_snp_init_locked(int * error)1280 static int __sev_snp_init_locked(int *error)
1281 {
1282 struct psp_device *psp = psp_master;
1283 struct sev_data_snp_init_ex data;
1284 struct sev_device *sev;
1285 void *arg = &data;
1286 int cmd, rc = 0;
1287
1288 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1289 return -ENODEV;
1290
1291 sev = psp->sev_data;
1292
1293 if (sev->snp_initialized)
1294 return 0;
1295
1296 if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) {
1297 dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n",
1298 SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR);
1299 return -EOPNOTSUPP;
1300 }
1301
1302 /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */
1303 on_each_cpu(snp_set_hsave_pa, NULL, 1);
1304
1305 /*
1306 * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
1307 * of system physical address ranges to convert into HV-fixed page
1308 * states during the RMP initialization. For instance, the memory that
1309 * UEFI reserves should be included in the that list. This allows system
1310 * components that occasionally write to memory (e.g. logging to UEFI
1311 * reserved regions) to not fail due to RMP initialization and SNP
1312 * enablement.
1313 *
1314 */
1315 if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) {
1316 /*
1317 * Firmware checks that the pages containing the ranges enumerated
1318 * in the RANGES structure are either in the default page state or in the
1319 * firmware page state.
1320 */
1321 snp_range_list = kzalloc(PAGE_SIZE, GFP_KERNEL);
1322 if (!snp_range_list) {
1323 dev_err(sev->dev,
1324 "SEV: SNP_INIT_EX range list memory allocation failed\n");
1325 return -ENOMEM;
1326 }
1327
1328 /*
1329 * Retrieve all reserved memory regions from the e820 memory map
1330 * to be setup as HV-fixed pages.
1331 */
1332 rc = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM, 0, ~0,
1333 snp_range_list, snp_filter_reserved_mem_regions);
1334 if (rc) {
1335 dev_err(sev->dev,
1336 "SEV: SNP_INIT_EX walk_iomem_res_desc failed rc = %d\n", rc);
1337 return rc;
1338 }
1339
1340 /*
1341 * Add HV_Fixed pages from other PSP sub-devices, such as SFS to the
1342 * HV_Fixed page list.
1343 */
1344 snp_add_hv_fixed_pages(sev, snp_range_list);
1345
1346 memset(&data, 0, sizeof(data));
1347 data.init_rmp = 1;
1348 data.list_paddr_en = 1;
1349 data.list_paddr = __psp_pa(snp_range_list);
1350 cmd = SEV_CMD_SNP_INIT_EX;
1351 } else {
1352 cmd = SEV_CMD_SNP_INIT;
1353 arg = NULL;
1354 }
1355
1356 /*
1357 * The following sequence must be issued before launching the first SNP
1358 * guest to ensure all dirty cache lines are flushed, including from
1359 * updates to the RMP table itself via the RMPUPDATE instruction:
1360 *
1361 * - WBINVD on all running CPUs
1362 * - SEV_CMD_SNP_INIT[_EX] firmware command
1363 * - WBINVD on all running CPUs
1364 * - SEV_CMD_SNP_DF_FLUSH firmware command
1365 */
1366 wbinvd_on_all_cpus();
1367
1368 rc = __sev_do_cmd_locked(cmd, arg, error);
1369 if (rc) {
1370 dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n",
1371 cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT",
1372 rc, *error);
1373 return rc;
1374 }
1375
1376 /* Prepare for first SNP guest launch after INIT. */
1377 wbinvd_on_all_cpus();
1378 rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error);
1379 if (rc) {
1380 dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n",
1381 rc, *error);
1382 return rc;
1383 }
1384
1385 snp_hv_fixed_pages_state_update(sev, HV_FIXED);
1386 sev->snp_initialized = true;
1387 dev_dbg(sev->dev, "SEV-SNP firmware initialized\n");
1388
1389 dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major,
1390 sev->api_minor, sev->build);
1391
1392 atomic_notifier_chain_register(&panic_notifier_list,
1393 &snp_panic_notifier);
1394
1395 sev_es_tmr_size = SNP_TMR_SIZE;
1396
1397 return 0;
1398 }
1399
__sev_platform_init_handle_tmr(struct sev_device * sev)1400 static void __sev_platform_init_handle_tmr(struct sev_device *sev)
1401 {
1402 if (sev_es_tmr)
1403 return;
1404
1405 /* Obtain the TMR memory area for SEV-ES use */
1406 sev_es_tmr = sev_fw_alloc(sev_es_tmr_size);
1407 if (sev_es_tmr) {
1408 /* Must flush the cache before giving it to the firmware */
1409 if (!sev->snp_initialized)
1410 clflush_cache_range(sev_es_tmr, sev_es_tmr_size);
1411 } else {
1412 dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n");
1413 }
1414 }
1415
1416 /*
1417 * If an init_ex_path is provided allocate a buffer for the file and
1418 * read in the contents. Additionally, if SNP is initialized, convert
1419 * the buffer pages to firmware pages.
1420 */
__sev_platform_init_handle_init_ex_path(struct sev_device * sev)1421 static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev)
1422 {
1423 struct page *page;
1424 int rc;
1425
1426 if (!init_ex_path)
1427 return 0;
1428
1429 if (sev_init_ex_buffer)
1430 return 0;
1431
1432 page = alloc_pages(GFP_KERNEL, get_order(NV_LENGTH));
1433 if (!page) {
1434 dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n");
1435 return -ENOMEM;
1436 }
1437
1438 sev_init_ex_buffer = page_address(page);
1439
1440 rc = sev_read_init_ex_file();
1441 if (rc)
1442 return rc;
1443
1444 /* If SEV-SNP is initialized, transition to firmware page. */
1445 if (sev->snp_initialized) {
1446 unsigned long npages;
1447
1448 npages = 1UL << get_order(NV_LENGTH);
1449 if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, false)) {
1450 dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n");
1451 return -ENOMEM;
1452 }
1453 }
1454
1455 return 0;
1456 }
1457
__sev_platform_init_locked(int * error)1458 static int __sev_platform_init_locked(int *error)
1459 {
1460 int rc, psp_ret, dfflush_error;
1461 struct sev_device *sev;
1462
1463 psp_ret = dfflush_error = SEV_RET_NO_FW_CALL;
1464
1465 if (!psp_master || !psp_master->sev_data)
1466 return -ENODEV;
1467
1468 sev = psp_master->sev_data;
1469
1470 if (sev->state == SEV_STATE_INIT)
1471 return 0;
1472
1473 __sev_platform_init_handle_tmr(sev);
1474
1475 rc = __sev_platform_init_handle_init_ex_path(sev);
1476 if (rc)
1477 return rc;
1478
1479 rc = __sev_do_init_locked(&psp_ret);
1480 if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
1481 /*
1482 * Initialization command returned an integrity check failure
1483 * status code, meaning that firmware load and validation of SEV
1484 * related persistent data has failed. Retrying the
1485 * initialization function should succeed by replacing the state
1486 * with a reset state.
1487 */
1488 dev_err(sev->dev,
1489 "SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state.");
1490 rc = __sev_do_init_locked(&psp_ret);
1491 }
1492
1493 if (error)
1494 *error = psp_ret;
1495
1496 if (rc) {
1497 dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n",
1498 sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc);
1499 return rc;
1500 }
1501
1502 sev->state = SEV_STATE_INIT;
1503
1504 /* Prepare for first SEV guest launch after INIT */
1505 wbinvd_on_all_cpus();
1506 rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, &dfflush_error);
1507 if (rc) {
1508 dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n",
1509 dfflush_error, rc);
1510 return rc;
1511 }
1512
1513 dev_dbg(sev->dev, "SEV firmware initialized\n");
1514
1515 dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
1516 sev->api_minor, sev->build);
1517
1518 return 0;
1519 }
1520
_sev_platform_init_locked(struct sev_platform_init_args * args)1521 static int _sev_platform_init_locked(struct sev_platform_init_args *args)
1522 {
1523 struct sev_device *sev;
1524 int rc;
1525
1526 if (!psp_master || !psp_master->sev_data)
1527 return -ENODEV;
1528
1529 sev = psp_master->sev_data;
1530
1531 if (sev->state == SEV_STATE_INIT)
1532 return 0;
1533
1534 rc = __sev_snp_init_locked(&args->error);
1535 if (rc && rc != -ENODEV)
1536 return rc;
1537
1538 /* Defer legacy SEV/SEV-ES support if allowed by caller/module. */
1539 if (args->probe && !psp_init_on_probe)
1540 return 0;
1541
1542 return __sev_platform_init_locked(&args->error);
1543 }
1544
sev_platform_init(struct sev_platform_init_args * args)1545 int sev_platform_init(struct sev_platform_init_args *args)
1546 {
1547 int rc;
1548
1549 mutex_lock(&sev_cmd_mutex);
1550 rc = _sev_platform_init_locked(args);
1551 mutex_unlock(&sev_cmd_mutex);
1552
1553 return rc;
1554 }
1555 EXPORT_SYMBOL_GPL(sev_platform_init);
1556
__sev_platform_shutdown_locked(int * error)1557 static int __sev_platform_shutdown_locked(int *error)
1558 {
1559 struct psp_device *psp = psp_master;
1560 struct sev_device *sev;
1561 int ret;
1562
1563 if (!psp || !psp->sev_data)
1564 return 0;
1565
1566 sev = psp->sev_data;
1567
1568 if (sev->state == SEV_STATE_UNINIT)
1569 return 0;
1570
1571 ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error);
1572 if (ret) {
1573 dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n",
1574 *error, ret);
1575 return ret;
1576 }
1577
1578 sev->state = SEV_STATE_UNINIT;
1579 dev_dbg(sev->dev, "SEV firmware shutdown\n");
1580
1581 return ret;
1582 }
1583
sev_get_platform_state(int * state,int * error)1584 static int sev_get_platform_state(int *state, int *error)
1585 {
1586 struct sev_user_data_status data;
1587 int rc;
1588
1589 rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, error);
1590 if (rc)
1591 return rc;
1592
1593 *state = data.state;
1594 return rc;
1595 }
1596
sev_move_to_init_state(struct sev_issue_cmd * argp,bool * shutdown_required)1597 static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
1598 {
1599 struct sev_platform_init_args init_args = {0};
1600 int rc;
1601
1602 rc = _sev_platform_init_locked(&init_args);
1603 if (rc) {
1604 argp->error = SEV_RET_INVALID_PLATFORM_STATE;
1605 return rc;
1606 }
1607
1608 *shutdown_required = true;
1609
1610 return 0;
1611 }
1612
snp_move_to_init_state(struct sev_issue_cmd * argp,bool * shutdown_required)1613 static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
1614 {
1615 int error, rc;
1616
1617 rc = __sev_snp_init_locked(&error);
1618 if (rc) {
1619 argp->error = SEV_RET_INVALID_PLATFORM_STATE;
1620 return rc;
1621 }
1622
1623 *shutdown_required = true;
1624
1625 return 0;
1626 }
1627
sev_ioctl_do_reset(struct sev_issue_cmd * argp,bool writable)1628 static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable)
1629 {
1630 int state, rc;
1631
1632 if (!writable)
1633 return -EPERM;
1634
1635 /*
1636 * The SEV spec requires that FACTORY_RESET must be issued in
1637 * UNINIT state. Before we go further lets check if any guest is
1638 * active.
1639 *
1640 * If FW is in WORKING state then deny the request otherwise issue
1641 * SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET.
1642 *
1643 */
1644 rc = sev_get_platform_state(&state, &argp->error);
1645 if (rc)
1646 return rc;
1647
1648 if (state == SEV_STATE_WORKING)
1649 return -EBUSY;
1650
1651 if (state == SEV_STATE_INIT) {
1652 rc = __sev_platform_shutdown_locked(&argp->error);
1653 if (rc)
1654 return rc;
1655 }
1656
1657 return __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error);
1658 }
1659
sev_ioctl_do_platform_status(struct sev_issue_cmd * argp)1660 static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp)
1661 {
1662 struct sev_user_data_status data;
1663 int ret;
1664
1665 memset(&data, 0, sizeof(data));
1666
1667 ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error);
1668 if (ret)
1669 return ret;
1670
1671 if (copy_to_user((void __user *)argp->data, &data, sizeof(data)))
1672 ret = -EFAULT;
1673
1674 return ret;
1675 }
1676
sev_ioctl_do_pek_pdh_gen(int cmd,struct sev_issue_cmd * argp,bool writable)1677 static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable)
1678 {
1679 struct sev_device *sev = psp_master->sev_data;
1680 bool shutdown_required = false;
1681 int rc;
1682
1683 if (!writable)
1684 return -EPERM;
1685
1686 if (sev->state == SEV_STATE_UNINIT) {
1687 rc = sev_move_to_init_state(argp, &shutdown_required);
1688 if (rc)
1689 return rc;
1690 }
1691
1692 rc = __sev_do_cmd_locked(cmd, NULL, &argp->error);
1693
1694 if (shutdown_required)
1695 __sev_firmware_shutdown(sev, false);
1696
1697 return rc;
1698 }
1699
sev_ioctl_do_pek_csr(struct sev_issue_cmd * argp,bool writable)1700 static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable)
1701 {
1702 struct sev_device *sev = psp_master->sev_data;
1703 struct sev_user_data_pek_csr input;
1704 bool shutdown_required = false;
1705 struct sev_data_pek_csr data;
1706 void __user *input_address;
1707 void *blob = NULL;
1708 int ret;
1709
1710 if (!writable)
1711 return -EPERM;
1712
1713 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
1714 return -EFAULT;
1715
1716 memset(&data, 0, sizeof(data));
1717
1718 /* userspace wants to query CSR length */
1719 if (!input.address || !input.length)
1720 goto cmd;
1721
1722 /* allocate a physically contiguous buffer to store the CSR blob */
1723 input_address = (void __user *)input.address;
1724 if (input.length > SEV_FW_BLOB_MAX_SIZE)
1725 return -EFAULT;
1726
1727 blob = kzalloc(input.length, GFP_KERNEL);
1728 if (!blob)
1729 return -ENOMEM;
1730
1731 data.address = __psp_pa(blob);
1732 data.len = input.length;
1733
1734 cmd:
1735 if (sev->state == SEV_STATE_UNINIT) {
1736 ret = sev_move_to_init_state(argp, &shutdown_required);
1737 if (ret)
1738 goto e_free_blob;
1739 }
1740
1741 ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error);
1742
1743 /* If we query the CSR length, FW responded with expected data. */
1744 input.length = data.len;
1745
1746 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
1747 ret = -EFAULT;
1748 goto e_free_blob;
1749 }
1750
1751 if (blob) {
1752 if (copy_to_user(input_address, blob, input.length))
1753 ret = -EFAULT;
1754 }
1755
1756 e_free_blob:
1757 if (shutdown_required)
1758 __sev_firmware_shutdown(sev, false);
1759
1760 kfree(blob);
1761 return ret;
1762 }
1763
psp_copy_user_blob(u64 uaddr,u32 len)1764 void *psp_copy_user_blob(u64 uaddr, u32 len)
1765 {
1766 if (!uaddr || !len)
1767 return ERR_PTR(-EINVAL);
1768
1769 /* verify that blob length does not exceed our limit */
1770 if (len > SEV_FW_BLOB_MAX_SIZE)
1771 return ERR_PTR(-EINVAL);
1772
1773 return memdup_user((void __user *)uaddr, len);
1774 }
1775 EXPORT_SYMBOL_GPL(psp_copy_user_blob);
1776
sev_get_api_version(void)1777 static int sev_get_api_version(void)
1778 {
1779 struct sev_device *sev = psp_master->sev_data;
1780 struct sev_user_data_status status;
1781 int error = 0, ret;
1782
1783 ret = sev_platform_status(&status, &error);
1784 if (ret) {
1785 dev_err(sev->dev,
1786 "SEV: failed to get status. Error: %#x\n", error);
1787 return 1;
1788 }
1789
1790 sev->api_major = status.api_major;
1791 sev->api_minor = status.api_minor;
1792 sev->build = status.build;
1793 sev->state = status.state;
1794
1795 return 0;
1796 }
1797
sev_get_firmware(struct device * dev,const struct firmware ** firmware)1798 static int sev_get_firmware(struct device *dev,
1799 const struct firmware **firmware)
1800 {
1801 char fw_name_specific[SEV_FW_NAME_SIZE];
1802 char fw_name_subset[SEV_FW_NAME_SIZE];
1803
1804 snprintf(fw_name_specific, sizeof(fw_name_specific),
1805 "amd/amd_sev_fam%.2xh_model%.2xh.sbin",
1806 boot_cpu_data.x86, boot_cpu_data.x86_model);
1807
1808 snprintf(fw_name_subset, sizeof(fw_name_subset),
1809 "amd/amd_sev_fam%.2xh_model%.1xxh.sbin",
1810 boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4);
1811
1812 /* Check for SEV FW for a particular model.
1813 * Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h
1814 *
1815 * or
1816 *
1817 * Check for SEV FW common to a subset of models.
1818 * Ex. amd_sev_fam17h_model0xh.sbin for
1819 * Family 17h Model 00h -- Family 17h Model 0Fh
1820 *
1821 * or
1822 *
1823 * Fall-back to using generic name: sev.fw
1824 */
1825 if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) ||
1826 (firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) ||
1827 (firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0))
1828 return 0;
1829
1830 return -ENOENT;
1831 }
1832
1833 /* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
sev_update_firmware(struct device * dev)1834 static int sev_update_firmware(struct device *dev)
1835 {
1836 struct sev_data_download_firmware *data;
1837 const struct firmware *firmware;
1838 int ret, error, order;
1839 struct page *p;
1840 u64 data_size;
1841
1842 if (!sev_version_greater_or_equal(0, 15)) {
1843 dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n");
1844 return -1;
1845 }
1846
1847 if (sev_get_firmware(dev, &firmware) == -ENOENT) {
1848 dev_dbg(dev, "No SEV firmware file present\n");
1849 return -1;
1850 }
1851
1852 /*
1853 * SEV FW expects the physical address given to it to be 32
1854 * byte aligned. Memory allocated has structure placed at the
1855 * beginning followed by the firmware being passed to the SEV
1856 * FW. Allocate enough memory for data structure + alignment
1857 * padding + SEV FW.
1858 */
1859 data_size = ALIGN(sizeof(struct sev_data_download_firmware), 32);
1860
1861 order = get_order(firmware->size + data_size);
1862 p = alloc_pages(GFP_KERNEL, order);
1863 if (!p) {
1864 ret = -1;
1865 goto fw_err;
1866 }
1867
1868 /*
1869 * Copy firmware data to a kernel allocated contiguous
1870 * memory region.
1871 */
1872 data = page_address(p);
1873 memcpy(page_address(p) + data_size, firmware->data, firmware->size);
1874
1875 data->address = __psp_pa(page_address(p) + data_size);
1876 data->len = firmware->size;
1877
1878 ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
1879
1880 /*
1881 * A quirk for fixing the committed TCB version, when upgrading from
1882 * earlier firmware version than 1.50.
1883 */
1884 if (!ret && !sev_version_greater_or_equal(1, 50))
1885 ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
1886
1887 if (ret)
1888 dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error);
1889
1890 __free_pages(p, order);
1891
1892 fw_err:
1893 release_firmware(firmware);
1894
1895 return ret;
1896 }
1897
__sev_snp_shutdown_locked(int * error,bool panic)1898 static int __sev_snp_shutdown_locked(int *error, bool panic)
1899 {
1900 struct psp_device *psp = psp_master;
1901 struct sev_device *sev;
1902 struct sev_data_snp_shutdown_ex data;
1903 int ret;
1904
1905 if (!psp || !psp->sev_data)
1906 return 0;
1907
1908 sev = psp->sev_data;
1909
1910 if (!sev->snp_initialized)
1911 return 0;
1912
1913 memset(&data, 0, sizeof(data));
1914 data.len = sizeof(data);
1915 data.iommu_snp_shutdown = 1;
1916
1917 /*
1918 * If invoked during panic handling, local interrupts are disabled
1919 * and all CPUs are stopped, so wbinvd_on_all_cpus() can't be called.
1920 * In that case, a wbinvd() is done on remote CPUs via the NMI
1921 * callback, so only a local wbinvd() is needed here.
1922 */
1923 if (!panic)
1924 wbinvd_on_all_cpus();
1925 else
1926 wbinvd();
1927
1928 ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error);
1929 /* SHUTDOWN may require DF_FLUSH */
1930 if (*error == SEV_RET_DFFLUSH_REQUIRED) {
1931 int dfflush_error = SEV_RET_NO_FW_CALL;
1932
1933 ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error);
1934 if (ret) {
1935 dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n",
1936 ret, dfflush_error);
1937 return ret;
1938 }
1939 /* reissue the shutdown command */
1940 ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data,
1941 error);
1942 }
1943 if (ret) {
1944 dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n",
1945 ret, *error);
1946 return ret;
1947 }
1948
1949 /*
1950 * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP
1951 * enforcement by the IOMMU and also transitions all pages
1952 * associated with the IOMMU to the Reclaim state.
1953 * Firmware was transitioning the IOMMU pages to Hypervisor state
1954 * before version 1.53. But, accounting for the number of assigned
1955 * 4kB pages in a 2M page was done incorrectly by not transitioning
1956 * to the Reclaim state. This resulted in RMP #PF when later accessing
1957 * the 2M page containing those pages during kexec boot. Hence, the
1958 * firmware now transitions these pages to Reclaim state and hypervisor
1959 * needs to transition these pages to shared state. SNP Firmware
1960 * version 1.53 and above are needed for kexec boot.
1961 */
1962 ret = amd_iommu_snp_disable();
1963 if (ret) {
1964 dev_err(sev->dev, "SNP IOMMU shutdown failed\n");
1965 return ret;
1966 }
1967
1968 snp_leak_hv_fixed_pages();
1969 sev->snp_initialized = false;
1970 dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n");
1971
1972 /*
1973 * __sev_snp_shutdown_locked() deadlocks when it tries to unregister
1974 * itself during panic as the panic notifier is called with RCU read
1975 * lock held and notifier unregistration does RCU synchronization.
1976 */
1977 if (!panic)
1978 atomic_notifier_chain_unregister(&panic_notifier_list,
1979 &snp_panic_notifier);
1980
1981 /* Reset TMR size back to default */
1982 sev_es_tmr_size = SEV_TMR_SIZE;
1983
1984 return ret;
1985 }
1986
sev_ioctl_do_pek_import(struct sev_issue_cmd * argp,bool writable)1987 static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable)
1988 {
1989 struct sev_device *sev = psp_master->sev_data;
1990 struct sev_user_data_pek_cert_import input;
1991 struct sev_data_pek_cert_import data;
1992 bool shutdown_required = false;
1993 void *pek_blob, *oca_blob;
1994 int ret;
1995
1996 if (!writable)
1997 return -EPERM;
1998
1999 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2000 return -EFAULT;
2001
2002 /* copy PEK certificate blobs from userspace */
2003 pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len);
2004 if (IS_ERR(pek_blob))
2005 return PTR_ERR(pek_blob);
2006
2007 data.reserved = 0;
2008 data.pek_cert_address = __psp_pa(pek_blob);
2009 data.pek_cert_len = input.pek_cert_len;
2010
2011 /* copy PEK certificate blobs from userspace */
2012 oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len);
2013 if (IS_ERR(oca_blob)) {
2014 ret = PTR_ERR(oca_blob);
2015 goto e_free_pek;
2016 }
2017
2018 data.oca_cert_address = __psp_pa(oca_blob);
2019 data.oca_cert_len = input.oca_cert_len;
2020
2021 /* If platform is not in INIT state then transition it to INIT */
2022 if (sev->state != SEV_STATE_INIT) {
2023 ret = sev_move_to_init_state(argp, &shutdown_required);
2024 if (ret)
2025 goto e_free_oca;
2026 }
2027
2028 ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error);
2029
2030 e_free_oca:
2031 if (shutdown_required)
2032 __sev_firmware_shutdown(sev, false);
2033
2034 kfree(oca_blob);
2035 e_free_pek:
2036 kfree(pek_blob);
2037 return ret;
2038 }
2039
sev_ioctl_do_get_id2(struct sev_issue_cmd * argp)2040 static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp)
2041 {
2042 struct sev_user_data_get_id2 input;
2043 struct sev_data_get_id data;
2044 void __user *input_address;
2045 void *id_blob = NULL;
2046 int ret;
2047
2048 /* SEV GET_ID is available from SEV API v0.16 and up */
2049 if (!sev_version_greater_or_equal(0, 16))
2050 return -ENOTSUPP;
2051
2052 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2053 return -EFAULT;
2054
2055 input_address = (void __user *)input.address;
2056
2057 if (input.address && input.length) {
2058 /*
2059 * The length of the ID shouldn't be assumed by software since
2060 * it may change in the future. The allocation size is limited
2061 * to 1 << (PAGE_SHIFT + MAX_PAGE_ORDER) by the page allocator.
2062 * If the allocation fails, simply return ENOMEM rather than
2063 * warning in the kernel log.
2064 */
2065 id_blob = kzalloc(input.length, GFP_KERNEL | __GFP_NOWARN);
2066 if (!id_blob)
2067 return -ENOMEM;
2068
2069 data.address = __psp_pa(id_blob);
2070 data.len = input.length;
2071 } else {
2072 data.address = 0;
2073 data.len = 0;
2074 }
2075
2076 ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, &data, &argp->error);
2077
2078 /*
2079 * Firmware will return the length of the ID value (either the minimum
2080 * required length or the actual length written), return it to the user.
2081 */
2082 input.length = data.len;
2083
2084 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2085 ret = -EFAULT;
2086 goto e_free;
2087 }
2088
2089 if (id_blob) {
2090 if (copy_to_user(input_address, id_blob, data.len)) {
2091 ret = -EFAULT;
2092 goto e_free;
2093 }
2094 }
2095
2096 e_free:
2097 kfree(id_blob);
2098
2099 return ret;
2100 }
2101
sev_ioctl_do_get_id(struct sev_issue_cmd * argp)2102 static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp)
2103 {
2104 struct sev_data_get_id *data;
2105 u64 data_size, user_size;
2106 void *id_blob, *mem;
2107 int ret;
2108
2109 /* SEV GET_ID available from SEV API v0.16 and up */
2110 if (!sev_version_greater_or_equal(0, 16))
2111 return -ENOTSUPP;
2112
2113 /* SEV FW expects the buffer it fills with the ID to be
2114 * 8-byte aligned. Memory allocated should be enough to
2115 * hold data structure + alignment padding + memory
2116 * where SEV FW writes the ID.
2117 */
2118 data_size = ALIGN(sizeof(struct sev_data_get_id), 8);
2119 user_size = sizeof(struct sev_user_data_get_id);
2120
2121 mem = kzalloc(data_size + user_size, GFP_KERNEL);
2122 if (!mem)
2123 return -ENOMEM;
2124
2125 data = mem;
2126 id_blob = mem + data_size;
2127
2128 data->address = __psp_pa(id_blob);
2129 data->len = user_size;
2130
2131 ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error);
2132 if (!ret) {
2133 if (copy_to_user((void __user *)argp->data, id_blob, data->len))
2134 ret = -EFAULT;
2135 }
2136
2137 kfree(mem);
2138
2139 return ret;
2140 }
2141
sev_ioctl_do_pdh_export(struct sev_issue_cmd * argp,bool writable)2142 static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
2143 {
2144 struct sev_device *sev = psp_master->sev_data;
2145 struct sev_user_data_pdh_cert_export input;
2146 void *pdh_blob = NULL, *cert_blob = NULL;
2147 struct sev_data_pdh_cert_export data;
2148 void __user *input_cert_chain_address;
2149 void __user *input_pdh_cert_address;
2150 bool shutdown_required = false;
2151 int ret;
2152
2153 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2154 return -EFAULT;
2155
2156 memset(&data, 0, sizeof(data));
2157
2158 input_pdh_cert_address = (void __user *)input.pdh_cert_address;
2159 input_cert_chain_address = (void __user *)input.cert_chain_address;
2160
2161 /* Userspace wants to query the certificate length. */
2162 if (!input.pdh_cert_address ||
2163 !input.pdh_cert_len ||
2164 !input.cert_chain_address)
2165 goto cmd;
2166
2167 /* Allocate a physically contiguous buffer to store the PDH blob. */
2168 if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE)
2169 return -EFAULT;
2170
2171 /* Allocate a physically contiguous buffer to store the cert chain blob. */
2172 if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE)
2173 return -EFAULT;
2174
2175 pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL);
2176 if (!pdh_blob)
2177 return -ENOMEM;
2178
2179 data.pdh_cert_address = __psp_pa(pdh_blob);
2180 data.pdh_cert_len = input.pdh_cert_len;
2181
2182 cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL);
2183 if (!cert_blob) {
2184 ret = -ENOMEM;
2185 goto e_free_pdh;
2186 }
2187
2188 data.cert_chain_address = __psp_pa(cert_blob);
2189 data.cert_chain_len = input.cert_chain_len;
2190
2191 cmd:
2192 /* If platform is not in INIT state then transition it to INIT. */
2193 if (sev->state != SEV_STATE_INIT) {
2194 if (!writable) {
2195 ret = -EPERM;
2196 goto e_free_cert;
2197 }
2198 ret = sev_move_to_init_state(argp, &shutdown_required);
2199 if (ret)
2200 goto e_free_cert;
2201 }
2202
2203 ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error);
2204
2205 /* If we query the length, FW responded with expected data. */
2206 input.cert_chain_len = data.cert_chain_len;
2207 input.pdh_cert_len = data.pdh_cert_len;
2208
2209 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2210 ret = -EFAULT;
2211 goto e_free_cert;
2212 }
2213
2214 if (pdh_blob) {
2215 if (copy_to_user(input_pdh_cert_address,
2216 pdh_blob, input.pdh_cert_len)) {
2217 ret = -EFAULT;
2218 goto e_free_cert;
2219 }
2220 }
2221
2222 if (cert_blob) {
2223 if (copy_to_user(input_cert_chain_address,
2224 cert_blob, input.cert_chain_len))
2225 ret = -EFAULT;
2226 }
2227
2228 e_free_cert:
2229 if (shutdown_required)
2230 __sev_firmware_shutdown(sev, false);
2231
2232 kfree(cert_blob);
2233 e_free_pdh:
2234 kfree(pdh_blob);
2235 return ret;
2236 }
2237
sev_ioctl_do_snp_platform_status(struct sev_issue_cmd * argp)2238 static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
2239 {
2240 struct sev_device *sev = psp_master->sev_data;
2241 bool shutdown_required = false;
2242 struct sev_data_snp_addr buf;
2243 struct page *status_page;
2244 int ret, error;
2245 void *data;
2246
2247 if (!argp->data)
2248 return -EINVAL;
2249
2250 status_page = alloc_page(GFP_KERNEL_ACCOUNT);
2251 if (!status_page)
2252 return -ENOMEM;
2253
2254 data = page_address(status_page);
2255
2256 if (!sev->snp_initialized) {
2257 ret = snp_move_to_init_state(argp, &shutdown_required);
2258 if (ret)
2259 goto cleanup;
2260 }
2261
2262 /*
2263 * Firmware expects status page to be in firmware-owned state, otherwise
2264 * it will report firmware error code INVALID_PAGE_STATE (0x1A).
2265 */
2266 if (rmp_mark_pages_firmware(__pa(data), 1, true)) {
2267 ret = -EFAULT;
2268 goto cleanup;
2269 }
2270
2271 buf.address = __psp_pa(data);
2272 ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error);
2273
2274 /*
2275 * Status page will be transitioned to Reclaim state upon success, or
2276 * left in Firmware state in failure. Use snp_reclaim_pages() to
2277 * transition either case back to Hypervisor-owned state.
2278 */
2279 if (snp_reclaim_pages(__pa(data), 1, true))
2280 return -EFAULT;
2281
2282 if (ret)
2283 goto cleanup;
2284
2285 if (copy_to_user((void __user *)argp->data, data,
2286 sizeof(struct sev_user_data_snp_status)))
2287 ret = -EFAULT;
2288
2289 cleanup:
2290 if (shutdown_required)
2291 __sev_snp_shutdown_locked(&error, false);
2292
2293 __free_pages(status_page, 0);
2294 return ret;
2295 }
2296
sev_ioctl_do_snp_commit(struct sev_issue_cmd * argp)2297 static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp)
2298 {
2299 struct sev_device *sev = psp_master->sev_data;
2300 struct sev_data_snp_commit buf;
2301 bool shutdown_required = false;
2302 int ret, error;
2303
2304 if (!sev->snp_initialized) {
2305 ret = snp_move_to_init_state(argp, &shutdown_required);
2306 if (ret)
2307 return ret;
2308 }
2309
2310 buf.len = sizeof(buf);
2311
2312 ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error);
2313
2314 if (shutdown_required)
2315 __sev_snp_shutdown_locked(&error, false);
2316
2317 return ret;
2318 }
2319
sev_ioctl_do_snp_set_config(struct sev_issue_cmd * argp,bool writable)2320 static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable)
2321 {
2322 struct sev_device *sev = psp_master->sev_data;
2323 struct sev_user_data_snp_config config;
2324 bool shutdown_required = false;
2325 int ret, error;
2326
2327 if (!argp->data)
2328 return -EINVAL;
2329
2330 if (!writable)
2331 return -EPERM;
2332
2333 if (copy_from_user(&config, (void __user *)argp->data, sizeof(config)))
2334 return -EFAULT;
2335
2336 if (!sev->snp_initialized) {
2337 ret = snp_move_to_init_state(argp, &shutdown_required);
2338 if (ret)
2339 return ret;
2340 }
2341
2342 ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error);
2343
2344 if (shutdown_required)
2345 __sev_snp_shutdown_locked(&error, false);
2346
2347 return ret;
2348 }
2349
sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd * argp,bool writable)2350 static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
2351 {
2352 struct sev_device *sev = psp_master->sev_data;
2353 struct sev_user_data_snp_vlek_load input;
2354 bool shutdown_required = false;
2355 int ret, error;
2356 void *blob;
2357
2358 if (!argp->data)
2359 return -EINVAL;
2360
2361 if (!writable)
2362 return -EPERM;
2363
2364 if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input)))
2365 return -EFAULT;
2366
2367 if (input.len != sizeof(input) || input.vlek_wrapped_version != 0)
2368 return -EINVAL;
2369
2370 blob = psp_copy_user_blob(input.vlek_wrapped_address,
2371 sizeof(struct sev_user_data_snp_wrapped_vlek_hashstick));
2372 if (IS_ERR(blob))
2373 return PTR_ERR(blob);
2374
2375 input.vlek_wrapped_address = __psp_pa(blob);
2376
2377 if (!sev->snp_initialized) {
2378 ret = snp_move_to_init_state(argp, &shutdown_required);
2379 if (ret)
2380 goto cleanup;
2381 }
2382
2383 ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error);
2384
2385 if (shutdown_required)
2386 __sev_snp_shutdown_locked(&error, false);
2387
2388 cleanup:
2389 kfree(blob);
2390
2391 return ret;
2392 }
2393
sev_ioctl(struct file * file,unsigned int ioctl,unsigned long arg)2394 static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
2395 {
2396 void __user *argp = (void __user *)arg;
2397 struct sev_issue_cmd input;
2398 int ret = -EFAULT;
2399 bool writable = file->f_mode & FMODE_WRITE;
2400
2401 if (!psp_master || !psp_master->sev_data)
2402 return -ENODEV;
2403
2404 if (ioctl != SEV_ISSUE_CMD)
2405 return -EINVAL;
2406
2407 if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd)))
2408 return -EFAULT;
2409
2410 if (input.cmd > SEV_MAX)
2411 return -EINVAL;
2412
2413 mutex_lock(&sev_cmd_mutex);
2414
2415 switch (input.cmd) {
2416
2417 case SEV_FACTORY_RESET:
2418 ret = sev_ioctl_do_reset(&input, writable);
2419 break;
2420 case SEV_PLATFORM_STATUS:
2421 ret = sev_ioctl_do_platform_status(&input);
2422 break;
2423 case SEV_PEK_GEN:
2424 ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input, writable);
2425 break;
2426 case SEV_PDH_GEN:
2427 ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input, writable);
2428 break;
2429 case SEV_PEK_CSR:
2430 ret = sev_ioctl_do_pek_csr(&input, writable);
2431 break;
2432 case SEV_PEK_CERT_IMPORT:
2433 ret = sev_ioctl_do_pek_import(&input, writable);
2434 break;
2435 case SEV_PDH_CERT_EXPORT:
2436 ret = sev_ioctl_do_pdh_export(&input, writable);
2437 break;
2438 case SEV_GET_ID:
2439 pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n");
2440 ret = sev_ioctl_do_get_id(&input);
2441 break;
2442 case SEV_GET_ID2:
2443 ret = sev_ioctl_do_get_id2(&input);
2444 break;
2445 case SNP_PLATFORM_STATUS:
2446 ret = sev_ioctl_do_snp_platform_status(&input);
2447 break;
2448 case SNP_COMMIT:
2449 ret = sev_ioctl_do_snp_commit(&input);
2450 break;
2451 case SNP_SET_CONFIG:
2452 ret = sev_ioctl_do_snp_set_config(&input, writable);
2453 break;
2454 case SNP_VLEK_LOAD:
2455 ret = sev_ioctl_do_snp_vlek_load(&input, writable);
2456 break;
2457 default:
2458 ret = -EINVAL;
2459 goto out;
2460 }
2461
2462 if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd)))
2463 ret = -EFAULT;
2464 out:
2465 mutex_unlock(&sev_cmd_mutex);
2466
2467 return ret;
2468 }
2469
2470 static const struct file_operations sev_fops = {
2471 .owner = THIS_MODULE,
2472 .unlocked_ioctl = sev_ioctl,
2473 };
2474
sev_platform_status(struct sev_user_data_status * data,int * error)2475 int sev_platform_status(struct sev_user_data_status *data, int *error)
2476 {
2477 return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error);
2478 }
2479 EXPORT_SYMBOL_GPL(sev_platform_status);
2480
sev_guest_deactivate(struct sev_data_deactivate * data,int * error)2481 int sev_guest_deactivate(struct sev_data_deactivate *data, int *error)
2482 {
2483 return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error);
2484 }
2485 EXPORT_SYMBOL_GPL(sev_guest_deactivate);
2486
sev_guest_activate(struct sev_data_activate * data,int * error)2487 int sev_guest_activate(struct sev_data_activate *data, int *error)
2488 {
2489 return sev_do_cmd(SEV_CMD_ACTIVATE, data, error);
2490 }
2491 EXPORT_SYMBOL_GPL(sev_guest_activate);
2492
sev_guest_decommission(struct sev_data_decommission * data,int * error)2493 int sev_guest_decommission(struct sev_data_decommission *data, int *error)
2494 {
2495 return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error);
2496 }
2497 EXPORT_SYMBOL_GPL(sev_guest_decommission);
2498
sev_guest_df_flush(int * error)2499 int sev_guest_df_flush(int *error)
2500 {
2501 return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error);
2502 }
2503 EXPORT_SYMBOL_GPL(sev_guest_df_flush);
2504
sev_exit(struct kref * ref)2505 static void sev_exit(struct kref *ref)
2506 {
2507 misc_deregister(&misc_dev->misc);
2508 kfree(misc_dev);
2509 misc_dev = NULL;
2510 }
2511
sev_misc_init(struct sev_device * sev)2512 static int sev_misc_init(struct sev_device *sev)
2513 {
2514 struct device *dev = sev->dev;
2515 int ret;
2516
2517 /*
2518 * SEV feature support can be detected on multiple devices but the SEV
2519 * FW commands must be issued on the master. During probe, we do not
2520 * know the master hence we create /dev/sev on the first device probe.
2521 * sev_do_cmd() finds the right master device to which to issue the
2522 * command to the firmware.
2523 */
2524 if (!misc_dev) {
2525 struct miscdevice *misc;
2526
2527 misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL);
2528 if (!misc_dev)
2529 return -ENOMEM;
2530
2531 misc = &misc_dev->misc;
2532 misc->minor = MISC_DYNAMIC_MINOR;
2533 misc->name = DEVICE_NAME;
2534 misc->fops = &sev_fops;
2535
2536 ret = misc_register(misc);
2537 if (ret)
2538 return ret;
2539
2540 kref_init(&misc_dev->refcount);
2541 } else {
2542 kref_get(&misc_dev->refcount);
2543 }
2544
2545 init_waitqueue_head(&sev->int_queue);
2546 sev->misc = misc_dev;
2547 dev_dbg(dev, "registered SEV device\n");
2548
2549 return 0;
2550 }
2551
sev_dev_init(struct psp_device * psp)2552 int sev_dev_init(struct psp_device *psp)
2553 {
2554 struct device *dev = psp->dev;
2555 struct sev_device *sev;
2556 int ret = -ENOMEM;
2557
2558 if (!boot_cpu_has(X86_FEATURE_SEV)) {
2559 dev_info_once(dev, "SEV: memory encryption not enabled by BIOS\n");
2560 return 0;
2561 }
2562
2563 sev = devm_kzalloc(dev, sizeof(*sev), GFP_KERNEL);
2564 if (!sev)
2565 goto e_err;
2566
2567 sev->cmd_buf = (void *)devm_get_free_pages(dev, GFP_KERNEL, 1);
2568 if (!sev->cmd_buf)
2569 goto e_sev;
2570
2571 sev->cmd_buf_backup = (uint8_t *)sev->cmd_buf + PAGE_SIZE;
2572
2573 psp->sev_data = sev;
2574
2575 sev->dev = dev;
2576 sev->psp = psp;
2577
2578 sev->io_regs = psp->io_regs;
2579
2580 sev->vdata = (struct sev_vdata *)psp->vdata->sev;
2581 if (!sev->vdata) {
2582 ret = -ENODEV;
2583 dev_err(dev, "sev: missing driver data\n");
2584 goto e_buf;
2585 }
2586
2587 psp_set_sev_irq_handler(psp, sev_irq_handler, sev);
2588
2589 ret = sev_misc_init(sev);
2590 if (ret)
2591 goto e_irq;
2592
2593 dev_notice(dev, "sev enabled\n");
2594
2595 return 0;
2596
2597 e_irq:
2598 psp_clear_sev_irq_handler(psp);
2599 e_buf:
2600 devm_free_pages(dev, (unsigned long)sev->cmd_buf);
2601 e_sev:
2602 devm_kfree(dev, sev);
2603 e_err:
2604 psp->sev_data = NULL;
2605
2606 dev_notice(dev, "sev initialization failed\n");
2607
2608 return ret;
2609 }
2610
__sev_firmware_shutdown(struct sev_device * sev,bool panic)2611 static void __sev_firmware_shutdown(struct sev_device *sev, bool panic)
2612 {
2613 int error;
2614
2615 __sev_platform_shutdown_locked(&error);
2616
2617 if (sev_es_tmr) {
2618 /*
2619 * The TMR area was encrypted, flush it from the cache.
2620 *
2621 * If invoked during panic handling, local interrupts are
2622 * disabled and all CPUs are stopped, so wbinvd_on_all_cpus()
2623 * can't be used. In that case, wbinvd() is done on remote CPUs
2624 * via the NMI callback, and done for this CPU later during
2625 * SNP shutdown, so wbinvd_on_all_cpus() can be skipped.
2626 */
2627 if (!panic)
2628 wbinvd_on_all_cpus();
2629
2630 __snp_free_firmware_pages(virt_to_page(sev_es_tmr),
2631 get_order(sev_es_tmr_size),
2632 true);
2633 sev_es_tmr = NULL;
2634 }
2635
2636 if (sev_init_ex_buffer) {
2637 __snp_free_firmware_pages(virt_to_page(sev_init_ex_buffer),
2638 get_order(NV_LENGTH),
2639 true);
2640 sev_init_ex_buffer = NULL;
2641 }
2642
2643 if (snp_range_list) {
2644 kfree(snp_range_list);
2645 snp_range_list = NULL;
2646 }
2647
2648 __sev_snp_shutdown_locked(&error, panic);
2649 }
2650
sev_firmware_shutdown(struct sev_device * sev)2651 static void sev_firmware_shutdown(struct sev_device *sev)
2652 {
2653 mutex_lock(&sev_cmd_mutex);
2654 __sev_firmware_shutdown(sev, false);
2655 mutex_unlock(&sev_cmd_mutex);
2656 }
2657
sev_platform_shutdown(void)2658 void sev_platform_shutdown(void)
2659 {
2660 if (!psp_master || !psp_master->sev_data)
2661 return;
2662
2663 sev_firmware_shutdown(psp_master->sev_data);
2664 }
2665 EXPORT_SYMBOL_GPL(sev_platform_shutdown);
2666
sev_dev_destroy(struct psp_device * psp)2667 void sev_dev_destroy(struct psp_device *psp)
2668 {
2669 struct sev_device *sev = psp->sev_data;
2670
2671 if (!sev)
2672 return;
2673
2674 sev_firmware_shutdown(sev);
2675
2676 if (sev->misc)
2677 kref_put(&misc_dev->refcount, sev_exit);
2678
2679 psp_clear_sev_irq_handler(psp);
2680 }
2681
snp_shutdown_on_panic(struct notifier_block * nb,unsigned long reason,void * arg)2682 static int snp_shutdown_on_panic(struct notifier_block *nb,
2683 unsigned long reason, void *arg)
2684 {
2685 struct sev_device *sev = psp_master->sev_data;
2686
2687 /*
2688 * If sev_cmd_mutex is already acquired, then it's likely
2689 * another PSP command is in flight and issuing a shutdown
2690 * would fail in unexpected ways. Rather than create even
2691 * more confusion during a panic, just bail out here.
2692 */
2693 if (mutex_is_locked(&sev_cmd_mutex))
2694 return NOTIFY_DONE;
2695
2696 __sev_firmware_shutdown(sev, true);
2697
2698 return NOTIFY_DONE;
2699 }
2700
sev_issue_cmd_external_user(struct file * filep,unsigned int cmd,void * data,int * error)2701 int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd,
2702 void *data, int *error)
2703 {
2704 if (!filep || filep->f_op != &sev_fops)
2705 return -EBADF;
2706
2707 return sev_do_cmd(cmd, data, error);
2708 }
2709 EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
2710
sev_pci_init(void)2711 void sev_pci_init(void)
2712 {
2713 struct sev_device *sev = psp_master->sev_data;
2714 u8 api_major, api_minor, build;
2715
2716 if (!sev)
2717 return;
2718
2719 psp_timeout = psp_probe_timeout;
2720
2721 if (sev_get_api_version())
2722 goto err;
2723
2724 api_major = sev->api_major;
2725 api_minor = sev->api_minor;
2726 build = sev->build;
2727
2728 if (sev_update_firmware(sev->dev) == 0)
2729 sev_get_api_version();
2730
2731 if (api_major != sev->api_major || api_minor != sev->api_minor ||
2732 build != sev->build)
2733 dev_info(sev->dev, "SEV firmware updated from %d.%d.%d to %d.%d.%d\n",
2734 api_major, api_minor, build,
2735 sev->api_major, sev->api_minor, sev->build);
2736
2737 return;
2738
2739 err:
2740 sev_dev_destroy(psp_master);
2741
2742 psp_master->sev_data = NULL;
2743 }
2744
sev_pci_exit(void)2745 void sev_pci_exit(void)
2746 {
2747 struct sev_device *sev = psp_master->sev_data;
2748
2749 if (!sev)
2750 return;
2751
2752 sev_firmware_shutdown(sev);
2753 }
2754