xref: /linux/drivers/crypto/ccp/sev-dev.c (revision 0eaee8fc9305b4ae8e75e68c30e7e93b6b367234)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * AMD Secure Encrypted Virtualization (SEV) interface
4  *
5  * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
6  *
7  * Author: Brijesh Singh <brijesh.singh@amd.com>
8  */
9 
10 #include <linux/bitfield.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/kthread.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/spinlock.h>
17 #include <linux/spinlock_types.h>
18 #include <linux/types.h>
19 #include <linux/mutex.h>
20 #include <linux/delay.h>
21 #include <linux/hw_random.h>
22 #include <linux/ccp.h>
23 #include <linux/firmware.h>
24 #include <linux/panic_notifier.h>
25 #include <linux/gfp.h>
26 #include <linux/cpufeature.h>
27 #include <linux/fs.h>
28 #include <linux/fs_struct.h>
29 #include <linux/psp.h>
30 #include <linux/amd-iommu.h>
31 #include <linux/crash_dump.h>
32 
33 #include <asm/smp.h>
34 #include <asm/cacheflush.h>
35 #include <asm/e820/types.h>
36 #include <asm/sev.h>
37 #include <asm/msr.h>
38 
39 #include "psp-dev.h"
40 #include "sev-dev.h"
41 
42 #define DEVICE_NAME		"sev"
43 #define SEV_FW_FILE		"amd/sev.fw"
44 #define SEV_FW_NAME_SIZE	64
45 
46 /* Minimum firmware version required for the SEV-SNP support */
47 #define SNP_MIN_API_MAJOR	1
48 #define SNP_MIN_API_MINOR	51
49 
50 /*
51  * Maximum number of firmware-writable buffers that might be specified
52  * in the parameters of a legacy SEV command buffer.
53  */
54 #define CMD_BUF_FW_WRITABLE_MAX 2
55 
56 /* Leave room in the descriptor array for an end-of-list indicator. */
57 #define CMD_BUF_DESC_MAX (CMD_BUF_FW_WRITABLE_MAX + 1)
58 
59 static DEFINE_MUTEX(sev_cmd_mutex);
60 static struct sev_misc_dev *misc_dev;
61 
62 static int psp_cmd_timeout = 100;
63 module_param(psp_cmd_timeout, int, 0644);
64 MODULE_PARM_DESC(psp_cmd_timeout, " default timeout value, in seconds, for PSP commands");
65 
66 static int psp_probe_timeout = 5;
67 module_param(psp_probe_timeout, int, 0644);
68 MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
69 
70 static char *init_ex_path;
71 module_param(init_ex_path, charp, 0444);
72 MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
73 
74 static bool psp_init_on_probe = true;
75 module_param(psp_init_on_probe, bool, 0444);
76 MODULE_PARM_DESC(psp_init_on_probe, "  if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
77 
78 #if IS_ENABLED(CONFIG_PCI_TSM)
79 static bool sev_tio_enabled = true;
80 module_param_named(tio, sev_tio_enabled, bool, 0444);
81 MODULE_PARM_DESC(tio, "Enables TIO in SNP_INIT_EX");
82 #else
83 static const bool sev_tio_enabled = false;
84 #endif
85 
86 MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
87 MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
88 MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
89 MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */
90 
91 static bool psp_dead;
92 static int psp_timeout;
93 
94 enum snp_hv_fixed_pages_state {
95 	ALLOCATED,
96 	HV_FIXED,
97 };
98 
99 struct snp_hv_fixed_pages_entry {
100 	struct list_head list;
101 	struct page *page;
102 	unsigned int order;
103 	bool free;
104 	enum snp_hv_fixed_pages_state page_state;
105 };
106 
107 static LIST_HEAD(snp_hv_fixed_pages);
108 
109 /* Trusted Memory Region (TMR):
110  *   The TMR is a 1MB area that must be 1MB aligned.  Use the page allocator
111  *   to allocate the memory, which will return aligned memory for the specified
112  *   allocation order.
113  *
114  * When SEV-SNP is enabled the TMR needs to be 2MB aligned and 2MB sized.
115  */
116 #define SEV_TMR_SIZE		(1024 * 1024)
117 #define SNP_TMR_SIZE		(2 * 1024 * 1024)
118 
119 static void *sev_es_tmr;
120 static size_t sev_es_tmr_size = SEV_TMR_SIZE;
121 
122 /* INIT_EX NV Storage:
123  *   The NV Storage is a 32Kb area and must be 4Kb page aligned.  Use the page
124  *   allocator to allocate the memory, which will return aligned memory for the
125  *   specified allocation order.
126  */
127 #define NV_LENGTH (32 * 1024)
128 static void *sev_init_ex_buffer;
129 
130 static void __sev_firmware_shutdown(struct sev_device *sev, bool panic);
131 
132 static int snp_shutdown_on_panic(struct notifier_block *nb,
133 				 unsigned long reason, void *arg);
134 
135 static struct notifier_block snp_panic_notifier = {
136 	.notifier_call = snp_shutdown_on_panic,
137 };
138 
139 static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
140 {
141 	struct sev_device *sev = psp_master->sev_data;
142 
143 	if (sev->api_major > maj)
144 		return true;
145 
146 	if (sev->api_major == maj && sev->api_minor >= min)
147 		return true;
148 
149 	return false;
150 }
151 
152 static void sev_irq_handler(int irq, void *data, unsigned int status)
153 {
154 	struct sev_device *sev = data;
155 	int reg;
156 
157 	/* Check if it is command completion: */
158 	if (!(status & SEV_CMD_COMPLETE))
159 		return;
160 
161 	/* Check if it is SEV command completion: */
162 	reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
163 	if (FIELD_GET(PSP_CMDRESP_RESP, reg)) {
164 		sev->int_rcvd = 1;
165 		wake_up(&sev->int_queue);
166 	}
167 }
168 
169 static int sev_wait_cmd_ioc(struct sev_device *sev,
170 			    unsigned int *reg, unsigned int timeout)
171 {
172 	int ret;
173 
174 	/*
175 	 * If invoked during panic handling, local interrupts are disabled,
176 	 * so the PSP command completion interrupt can't be used. Poll for
177 	 * PSP command completion instead.
178 	 */
179 	if (irqs_disabled()) {
180 		unsigned long timeout_usecs = (timeout * USEC_PER_SEC) / 10;
181 
182 		/* Poll for SEV command completion: */
183 		while (timeout_usecs--) {
184 			*reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
185 			if (*reg & PSP_CMDRESP_RESP)
186 				return 0;
187 
188 			udelay(10);
189 		}
190 		return -ETIMEDOUT;
191 	}
192 
193 	ret = wait_event_timeout(sev->int_queue,
194 			sev->int_rcvd, timeout * HZ);
195 	if (!ret)
196 		return -ETIMEDOUT;
197 
198 	*reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
199 
200 	return 0;
201 }
202 
203 static int sev_cmd_buffer_len(int cmd)
204 {
205 	switch (cmd) {
206 	case SEV_CMD_INIT:			return sizeof(struct sev_data_init);
207 	case SEV_CMD_INIT_EX:                   return sizeof(struct sev_data_init_ex);
208 	case SEV_CMD_SNP_SHUTDOWN_EX:		return sizeof(struct sev_data_snp_shutdown_ex);
209 	case SEV_CMD_SNP_INIT_EX:		return sizeof(struct sev_data_snp_init_ex);
210 	case SEV_CMD_PLATFORM_STATUS:		return sizeof(struct sev_user_data_status);
211 	case SEV_CMD_PEK_CSR:			return sizeof(struct sev_data_pek_csr);
212 	case SEV_CMD_PEK_CERT_IMPORT:		return sizeof(struct sev_data_pek_cert_import);
213 	case SEV_CMD_PDH_CERT_EXPORT:		return sizeof(struct sev_data_pdh_cert_export);
214 	case SEV_CMD_LAUNCH_START:		return sizeof(struct sev_data_launch_start);
215 	case SEV_CMD_LAUNCH_UPDATE_DATA:	return sizeof(struct sev_data_launch_update_data);
216 	case SEV_CMD_LAUNCH_UPDATE_VMSA:	return sizeof(struct sev_data_launch_update_vmsa);
217 	case SEV_CMD_LAUNCH_FINISH:		return sizeof(struct sev_data_launch_finish);
218 	case SEV_CMD_LAUNCH_MEASURE:		return sizeof(struct sev_data_launch_measure);
219 	case SEV_CMD_ACTIVATE:			return sizeof(struct sev_data_activate);
220 	case SEV_CMD_DEACTIVATE:		return sizeof(struct sev_data_deactivate);
221 	case SEV_CMD_DECOMMISSION:		return sizeof(struct sev_data_decommission);
222 	case SEV_CMD_GUEST_STATUS:		return sizeof(struct sev_data_guest_status);
223 	case SEV_CMD_DBG_DECRYPT:		return sizeof(struct sev_data_dbg);
224 	case SEV_CMD_DBG_ENCRYPT:		return sizeof(struct sev_data_dbg);
225 	case SEV_CMD_SEND_START:		return sizeof(struct sev_data_send_start);
226 	case SEV_CMD_SEND_UPDATE_DATA:		return sizeof(struct sev_data_send_update_data);
227 	case SEV_CMD_SEND_UPDATE_VMSA:		return sizeof(struct sev_data_send_update_vmsa);
228 	case SEV_CMD_SEND_FINISH:		return sizeof(struct sev_data_send_finish);
229 	case SEV_CMD_RECEIVE_START:		return sizeof(struct sev_data_receive_start);
230 	case SEV_CMD_RECEIVE_FINISH:		return sizeof(struct sev_data_receive_finish);
231 	case SEV_CMD_RECEIVE_UPDATE_DATA:	return sizeof(struct sev_data_receive_update_data);
232 	case SEV_CMD_RECEIVE_UPDATE_VMSA:	return sizeof(struct sev_data_receive_update_vmsa);
233 	case SEV_CMD_LAUNCH_UPDATE_SECRET:	return sizeof(struct sev_data_launch_secret);
234 	case SEV_CMD_DOWNLOAD_FIRMWARE:		return sizeof(struct sev_data_download_firmware);
235 	case SEV_CMD_GET_ID:			return sizeof(struct sev_data_get_id);
236 	case SEV_CMD_ATTESTATION_REPORT:	return sizeof(struct sev_data_attestation_report);
237 	case SEV_CMD_SEND_CANCEL:		return sizeof(struct sev_data_send_cancel);
238 	case SEV_CMD_SNP_GCTX_CREATE:		return sizeof(struct sev_data_snp_addr);
239 	case SEV_CMD_SNP_LAUNCH_START:		return sizeof(struct sev_data_snp_launch_start);
240 	case SEV_CMD_SNP_LAUNCH_UPDATE:		return sizeof(struct sev_data_snp_launch_update);
241 	case SEV_CMD_SNP_ACTIVATE:		return sizeof(struct sev_data_snp_activate);
242 	case SEV_CMD_SNP_DECOMMISSION:		return sizeof(struct sev_data_snp_addr);
243 	case SEV_CMD_SNP_PAGE_RECLAIM:		return sizeof(struct sev_data_snp_page_reclaim);
244 	case SEV_CMD_SNP_GUEST_STATUS:		return sizeof(struct sev_data_snp_guest_status);
245 	case SEV_CMD_SNP_LAUNCH_FINISH:		return sizeof(struct sev_data_snp_launch_finish);
246 	case SEV_CMD_SNP_DBG_DECRYPT:		return sizeof(struct sev_data_snp_dbg);
247 	case SEV_CMD_SNP_DBG_ENCRYPT:		return sizeof(struct sev_data_snp_dbg);
248 	case SEV_CMD_SNP_VERIFY_MITIGATION:	return sizeof(struct sev_data_snp_verify_mitigation);
249 	case SEV_CMD_SNP_PAGE_UNSMASH:		return sizeof(struct sev_data_snp_page_unsmash);
250 	case SEV_CMD_SNP_PLATFORM_STATUS:	return sizeof(struct sev_data_snp_addr);
251 	case SEV_CMD_SNP_GUEST_REQUEST:		return sizeof(struct sev_data_snp_guest_request);
252 	case SEV_CMD_SNP_CONFIG:		return sizeof(struct sev_user_data_snp_config);
253 	case SEV_CMD_SNP_COMMIT:		return sizeof(struct sev_data_snp_commit);
254 	case SEV_CMD_SNP_FEATURE_INFO:		return sizeof(struct sev_data_snp_feature_info);
255 	case SEV_CMD_SNP_VLEK_LOAD:		return sizeof(struct sev_user_data_snp_vlek_load);
256 	default:				return sev_tio_cmd_buffer_len(cmd);
257 	}
258 
259 	return 0;
260 }
261 
262 static struct file *open_file_as_root(const char *filename, int flags, umode_t mode)
263 {
264 	CLASS(prepare_creds, cred)();
265 	if (!cred)
266 		return ERR_PTR(-ENOMEM);
267 
268 	cred->fsuid = GLOBAL_ROOT_UID;
269 
270 	scoped_with_init_fs() {
271 		scoped_with_creds(cred)
272 			return filp_open(filename, flags, mode);
273 	}
274 }
275 
276 static int sev_read_init_ex_file(void)
277 {
278 	struct sev_device *sev = psp_master->sev_data;
279 	struct file *fp;
280 	ssize_t nread;
281 
282 	lockdep_assert_held(&sev_cmd_mutex);
283 
284 	if (!sev_init_ex_buffer)
285 		return -EOPNOTSUPP;
286 
287 	fp = open_file_as_root(init_ex_path, O_RDONLY, 0);
288 	if (IS_ERR(fp)) {
289 		int ret = PTR_ERR(fp);
290 
291 		if (ret == -ENOENT) {
292 			dev_info(sev->dev,
293 				"SEV: %s does not exist and will be created later.\n",
294 				init_ex_path);
295 			ret = 0;
296 		} else {
297 			dev_err(sev->dev,
298 				"SEV: could not open %s for read, error %d\n",
299 				init_ex_path, ret);
300 		}
301 		return ret;
302 	}
303 
304 	nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
305 	if (nread != NV_LENGTH) {
306 		dev_info(sev->dev,
307 			"SEV: could not read %u bytes to non volatile memory area, ret %ld\n",
308 			NV_LENGTH, nread);
309 	}
310 
311 	dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
312 	filp_close(fp, NULL);
313 
314 	return 0;
315 }
316 
317 static int sev_write_init_ex_file(void)
318 {
319 	struct sev_device *sev = psp_master->sev_data;
320 	struct file *fp;
321 	loff_t offset = 0;
322 	ssize_t nwrite;
323 
324 	lockdep_assert_held(&sev_cmd_mutex);
325 
326 	if (!sev_init_ex_buffer)
327 		return 0;
328 
329 	fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600);
330 	if (IS_ERR(fp)) {
331 		int ret = PTR_ERR(fp);
332 
333 		dev_err(sev->dev,
334 			"SEV: could not open file for write, error %d\n",
335 			ret);
336 		return ret;
337 	}
338 
339 	nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
340 	vfs_fsync(fp, 0);
341 	filp_close(fp, NULL);
342 
343 	if (nwrite != NV_LENGTH) {
344 		dev_err(sev->dev,
345 			"SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
346 			NV_LENGTH, nwrite);
347 		return -EIO;
348 	}
349 
350 	dev_dbg(sev->dev, "SEV: write successful to NV file\n");
351 
352 	return 0;
353 }
354 
355 static int sev_write_init_ex_file_if_required(int cmd_id)
356 {
357 	lockdep_assert_held(&sev_cmd_mutex);
358 
359 	if (!sev_init_ex_buffer)
360 		return 0;
361 
362 	/*
363 	 * Only a few platform commands modify the SPI/NV area, but none of the
364 	 * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
365 	 * PEK_CERT_IMPORT, and PDH_GEN do.
366 	 */
367 	switch (cmd_id) {
368 	case SEV_CMD_FACTORY_RESET:
369 	case SEV_CMD_INIT_EX:
370 	case SEV_CMD_PDH_GEN:
371 	case SEV_CMD_PEK_CERT_IMPORT:
372 	case SEV_CMD_PEK_GEN:
373 		break;
374 	default:
375 		return 0;
376 	}
377 
378 	return sev_write_init_ex_file();
379 }
380 
381 int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked)
382 {
383 	int ret, err, i;
384 
385 	paddr = __sme_clr(ALIGN_DOWN(paddr, PAGE_SIZE));
386 
387 	for (i = 0; i < npages; i++, paddr += PAGE_SIZE) {
388 		struct sev_data_snp_page_reclaim data = {0};
389 
390 		data.paddr = paddr;
391 
392 		if (locked)
393 			ret = __sev_do_cmd_locked(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
394 		else
395 			ret = sev_do_cmd(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
396 
397 		if (ret)
398 			goto cleanup;
399 
400 		ret = rmp_make_shared(__phys_to_pfn(paddr), PG_LEVEL_4K);
401 		if (ret)
402 			goto cleanup;
403 	}
404 
405 	return 0;
406 
407 cleanup:
408 	/*
409 	 * If there was a failure reclaiming the page then it is no longer safe
410 	 * to release it back to the system; leak it instead.
411 	 */
412 	snp_leak_pages(__phys_to_pfn(paddr), npages - i);
413 	return ret;
414 }
415 EXPORT_SYMBOL_GPL(snp_reclaim_pages);
416 
417 static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked)
418 {
419 	unsigned long pfn = __sme_clr(paddr) >> PAGE_SHIFT;
420 	int rc, i;
421 
422 	for (i = 0; i < npages; i++, pfn++) {
423 		rc = rmp_make_private(pfn, 0, PG_LEVEL_4K, 0, true);
424 		if (rc)
425 			goto cleanup;
426 	}
427 
428 	return 0;
429 
430 cleanup:
431 	/*
432 	 * Try unrolling the firmware state changes by
433 	 * reclaiming the pages which were already changed to the
434 	 * firmware state.
435 	 */
436 	snp_reclaim_pages(paddr, i, locked);
437 
438 	return rc;
439 }
440 
441 static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order, bool locked)
442 {
443 	unsigned long npages = 1ul << order, paddr;
444 	struct sev_device *sev;
445 	struct page *page;
446 
447 	if (!psp_master || !psp_master->sev_data)
448 		return NULL;
449 
450 	page = alloc_pages(gfp_mask, order);
451 	if (!page)
452 		return NULL;
453 
454 	/* If SEV-SNP is initialized then add the page in RMP table. */
455 	sev = psp_master->sev_data;
456 	if (!sev->snp_initialized)
457 		return page;
458 
459 	paddr = __pa((unsigned long)page_address(page));
460 	if (rmp_mark_pages_firmware(paddr, npages, locked))
461 		return NULL;
462 
463 	return page;
464 }
465 
466 void *snp_alloc_firmware_page(gfp_t gfp_mask)
467 {
468 	struct page *page;
469 
470 	page = __snp_alloc_firmware_pages(gfp_mask, 0, false);
471 
472 	return page ? page_address(page) : NULL;
473 }
474 EXPORT_SYMBOL_GPL(snp_alloc_firmware_page);
475 
476 static void __snp_free_firmware_pages(struct page *page, int order, bool locked)
477 {
478 	struct sev_device *sev = psp_master->sev_data;
479 	unsigned long paddr, npages = 1ul << order;
480 
481 	if (!page)
482 		return;
483 
484 	paddr = __pa((unsigned long)page_address(page));
485 	if (sev->snp_initialized &&
486 	    snp_reclaim_pages(paddr, npages, locked))
487 		return;
488 
489 	__free_pages(page, order);
490 }
491 
492 void snp_free_firmware_page(void *addr)
493 {
494 	if (!addr)
495 		return;
496 
497 	__snp_free_firmware_pages(virt_to_page(addr), 0, false);
498 }
499 EXPORT_SYMBOL_GPL(snp_free_firmware_page);
500 
501 static void *sev_fw_alloc(unsigned long len)
502 {
503 	struct page *page;
504 
505 	page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len), true);
506 	if (!page)
507 		return NULL;
508 
509 	return page_address(page);
510 }
511 
512 /**
513  * struct cmd_buf_desc - descriptors for managing legacy SEV command address
514  * parameters corresponding to buffers that may be written to by firmware.
515  *
516  * @paddr_ptr:  pointer to the address parameter in the command buffer which may
517  *              need to be saved/restored depending on whether a bounce buffer
518  *              is used. In the case of a bounce buffer, the command buffer
519  *              needs to be updated with the address of the new bounce buffer
520  *              snp_map_cmd_buf_desc() has allocated specifically for it. Must
521  *              be NULL if this descriptor is only an end-of-list indicator.
522  *
523  * @paddr_orig: storage for the original address parameter, which can be used to
524  *              restore the original value in @paddr_ptr in cases where it is
525  *              replaced with the address of a bounce buffer.
526  *
527  * @len: length of buffer located at the address originally stored at @paddr_ptr
528  *
529  * @guest_owned: true if the address corresponds to guest-owned pages, in which
530  *               case bounce buffers are not needed.
531  */
532 struct cmd_buf_desc {
533 	u64 *paddr_ptr;
534 	u64 paddr_orig;
535 	u32 len;
536 	bool guest_owned;
537 };
538 
539 /*
540  * If a legacy SEV command parameter is a memory address, those pages in
541  * turn need to be transitioned to/from firmware-owned before/after
542  * executing the firmware command.
543  *
544  * Additionally, in cases where those pages are not guest-owned, a bounce
545  * buffer is needed in place of the original memory address parameter.
546  *
547  * A set of descriptors are used to keep track of this handling, and
548  * initialized here based on the specific commands being executed.
549  */
550 static void snp_populate_cmd_buf_desc_list(int cmd, void *cmd_buf,
551 					   struct cmd_buf_desc *desc_list)
552 {
553 	switch (cmd) {
554 	case SEV_CMD_PDH_CERT_EXPORT: {
555 		struct sev_data_pdh_cert_export *data = cmd_buf;
556 
557 		desc_list[0].paddr_ptr = &data->pdh_cert_address;
558 		desc_list[0].len = data->pdh_cert_len;
559 		desc_list[1].paddr_ptr = &data->cert_chain_address;
560 		desc_list[1].len = data->cert_chain_len;
561 		break;
562 	}
563 	case SEV_CMD_GET_ID: {
564 		struct sev_data_get_id *data = cmd_buf;
565 
566 		desc_list[0].paddr_ptr = &data->address;
567 		desc_list[0].len = data->len;
568 		break;
569 	}
570 	case SEV_CMD_PEK_CSR: {
571 		struct sev_data_pek_csr *data = cmd_buf;
572 
573 		desc_list[0].paddr_ptr = &data->address;
574 		desc_list[0].len = data->len;
575 		break;
576 	}
577 	case SEV_CMD_LAUNCH_UPDATE_DATA: {
578 		struct sev_data_launch_update_data *data = cmd_buf;
579 
580 		desc_list[0].paddr_ptr = &data->address;
581 		desc_list[0].len = data->len;
582 		desc_list[0].guest_owned = true;
583 		break;
584 	}
585 	case SEV_CMD_LAUNCH_UPDATE_VMSA: {
586 		struct sev_data_launch_update_vmsa *data = cmd_buf;
587 
588 		desc_list[0].paddr_ptr = &data->address;
589 		desc_list[0].len = data->len;
590 		desc_list[0].guest_owned = true;
591 		break;
592 	}
593 	case SEV_CMD_LAUNCH_MEASURE: {
594 		struct sev_data_launch_measure *data = cmd_buf;
595 
596 		desc_list[0].paddr_ptr = &data->address;
597 		desc_list[0].len = data->len;
598 		break;
599 	}
600 	case SEV_CMD_LAUNCH_UPDATE_SECRET: {
601 		struct sev_data_launch_secret *data = cmd_buf;
602 
603 		desc_list[0].paddr_ptr = &data->guest_address;
604 		desc_list[0].len = data->guest_len;
605 		desc_list[0].guest_owned = true;
606 		break;
607 	}
608 	case SEV_CMD_DBG_DECRYPT: {
609 		struct sev_data_dbg *data = cmd_buf;
610 
611 		desc_list[0].paddr_ptr = &data->dst_addr;
612 		desc_list[0].len = data->len;
613 		desc_list[0].guest_owned = true;
614 		break;
615 	}
616 	case SEV_CMD_DBG_ENCRYPT: {
617 		struct sev_data_dbg *data = cmd_buf;
618 
619 		desc_list[0].paddr_ptr = &data->dst_addr;
620 		desc_list[0].len = data->len;
621 		desc_list[0].guest_owned = true;
622 		break;
623 	}
624 	case SEV_CMD_ATTESTATION_REPORT: {
625 		struct sev_data_attestation_report *data = cmd_buf;
626 
627 		desc_list[0].paddr_ptr = &data->address;
628 		desc_list[0].len = data->len;
629 		break;
630 	}
631 	case SEV_CMD_SEND_START: {
632 		struct sev_data_send_start *data = cmd_buf;
633 
634 		desc_list[0].paddr_ptr = &data->session_address;
635 		desc_list[0].len = data->session_len;
636 		break;
637 	}
638 	case SEV_CMD_SEND_UPDATE_DATA: {
639 		struct sev_data_send_update_data *data = cmd_buf;
640 
641 		desc_list[0].paddr_ptr = &data->hdr_address;
642 		desc_list[0].len = data->hdr_len;
643 		desc_list[1].paddr_ptr = &data->trans_address;
644 		desc_list[1].len = data->trans_len;
645 		break;
646 	}
647 	case SEV_CMD_SEND_UPDATE_VMSA: {
648 		struct sev_data_send_update_vmsa *data = cmd_buf;
649 
650 		desc_list[0].paddr_ptr = &data->hdr_address;
651 		desc_list[0].len = data->hdr_len;
652 		desc_list[1].paddr_ptr = &data->trans_address;
653 		desc_list[1].len = data->trans_len;
654 		break;
655 	}
656 	case SEV_CMD_RECEIVE_UPDATE_DATA: {
657 		struct sev_data_receive_update_data *data = cmd_buf;
658 
659 		desc_list[0].paddr_ptr = &data->guest_address;
660 		desc_list[0].len = data->guest_len;
661 		desc_list[0].guest_owned = true;
662 		break;
663 	}
664 	case SEV_CMD_RECEIVE_UPDATE_VMSA: {
665 		struct sev_data_receive_update_vmsa *data = cmd_buf;
666 
667 		desc_list[0].paddr_ptr = &data->guest_address;
668 		desc_list[0].len = data->guest_len;
669 		desc_list[0].guest_owned = true;
670 		break;
671 	}
672 	default:
673 		break;
674 	}
675 }
676 
677 static int snp_map_cmd_buf_desc(struct cmd_buf_desc *desc)
678 {
679 	unsigned int npages;
680 
681 	if (!desc->len)
682 		return 0;
683 
684 	/* Allocate a bounce buffer if this isn't a guest owned page. */
685 	if (!desc->guest_owned) {
686 		struct page *page;
687 
688 		page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(desc->len));
689 		if (!page) {
690 			pr_warn("Failed to allocate bounce buffer for SEV legacy command.\n");
691 			return -ENOMEM;
692 		}
693 
694 		desc->paddr_orig = *desc->paddr_ptr;
695 		*desc->paddr_ptr = __psp_pa(page_to_virt(page));
696 	}
697 
698 	npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
699 
700 	/* Transition the buffer to firmware-owned. */
701 	if (rmp_mark_pages_firmware(*desc->paddr_ptr, npages, true)) {
702 		pr_warn("Error moving pages to firmware-owned state for SEV legacy command.\n");
703 		return -EFAULT;
704 	}
705 
706 	return 0;
707 }
708 
709 static int snp_unmap_cmd_buf_desc(struct cmd_buf_desc *desc)
710 {
711 	unsigned int npages;
712 
713 	if (!desc->len)
714 		return 0;
715 
716 	npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
717 
718 	/* Transition the buffers back to hypervisor-owned. */
719 	if (snp_reclaim_pages(*desc->paddr_ptr, npages, true)) {
720 		pr_warn("Failed to reclaim firmware-owned pages while issuing SEV legacy command.\n");
721 		return -EFAULT;
722 	}
723 
724 	/* Copy data from bounce buffer and then free it. */
725 	if (!desc->guest_owned) {
726 		void *bounce_buf = __va(__sme_clr(*desc->paddr_ptr));
727 		void *dst_buf = __va(__sme_clr(desc->paddr_orig));
728 
729 		memcpy(dst_buf, bounce_buf, desc->len);
730 		__free_pages(virt_to_page(bounce_buf), get_order(desc->len));
731 
732 		/* Restore the original address in the command buffer. */
733 		*desc->paddr_ptr = desc->paddr_orig;
734 	}
735 
736 	return 0;
737 }
738 
739 static int snp_map_cmd_buf_desc_list(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
740 {
741 	int i;
742 
743 	snp_populate_cmd_buf_desc_list(cmd, cmd_buf, desc_list);
744 
745 	for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
746 		struct cmd_buf_desc *desc = &desc_list[i];
747 
748 		if (!desc->paddr_ptr)
749 			break;
750 
751 		if (snp_map_cmd_buf_desc(desc))
752 			goto err_unmap;
753 	}
754 
755 	return 0;
756 
757 err_unmap:
758 	for (i--; i >= 0; i--)
759 		snp_unmap_cmd_buf_desc(&desc_list[i]);
760 
761 	return -EFAULT;
762 }
763 
764 static int snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc *desc_list)
765 {
766 	int i, ret = 0;
767 
768 	for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
769 		struct cmd_buf_desc *desc = &desc_list[i];
770 
771 		if (!desc->paddr_ptr)
772 			break;
773 
774 		if (snp_unmap_cmd_buf_desc(&desc_list[i]))
775 			ret = -EFAULT;
776 	}
777 
778 	return ret;
779 }
780 
781 static bool sev_cmd_buf_writable(int cmd)
782 {
783 	switch (cmd) {
784 	case SEV_CMD_PLATFORM_STATUS:
785 	case SEV_CMD_GUEST_STATUS:
786 	case SEV_CMD_LAUNCH_START:
787 	case SEV_CMD_RECEIVE_START:
788 	case SEV_CMD_LAUNCH_MEASURE:
789 	case SEV_CMD_SEND_START:
790 	case SEV_CMD_SEND_UPDATE_DATA:
791 	case SEV_CMD_SEND_UPDATE_VMSA:
792 	case SEV_CMD_PEK_CSR:
793 	case SEV_CMD_PDH_CERT_EXPORT:
794 	case SEV_CMD_GET_ID:
795 	case SEV_CMD_ATTESTATION_REPORT:
796 		return true;
797 	default:
798 		return false;
799 	}
800 }
801 
802 /* After SNP is INIT'ed, the behavior of legacy SEV commands is changed. */
803 static bool snp_legacy_handling_needed(int cmd)
804 {
805 	struct sev_device *sev = psp_master->sev_data;
806 
807 	return cmd < SEV_CMD_SNP_INIT && sev->snp_initialized;
808 }
809 
810 static int snp_prep_cmd_buf(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
811 {
812 	if (!snp_legacy_handling_needed(cmd))
813 		return 0;
814 
815 	if (snp_map_cmd_buf_desc_list(cmd, cmd_buf, desc_list))
816 		return -EFAULT;
817 
818 	/*
819 	 * Before command execution, the command buffer needs to be put into
820 	 * the firmware-owned state.
821 	 */
822 	if (sev_cmd_buf_writable(cmd)) {
823 		if (rmp_mark_pages_firmware(__pa(cmd_buf), 1, true))
824 			return -EFAULT;
825 	}
826 
827 	return 0;
828 }
829 
830 static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf)
831 {
832 	if (!snp_legacy_handling_needed(cmd))
833 		return 0;
834 
835 	/*
836 	 * After command completion, the command buffer needs to be put back
837 	 * into the hypervisor-owned state.
838 	 */
839 	if (sev_cmd_buf_writable(cmd))
840 		if (snp_reclaim_pages(__pa(cmd_buf), 1, true))
841 			return -EFAULT;
842 
843 	return 0;
844 }
845 
846 int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
847 {
848 	struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0};
849 	struct psp_device *psp = psp_master;
850 	struct sev_device *sev;
851 	unsigned int cmdbuff_hi, cmdbuff_lo;
852 	unsigned int phys_lsb, phys_msb;
853 	unsigned int reg;
854 	void *cmd_buf;
855 	int buf_len;
856 	int ret = 0;
857 
858 	if (!psp || !psp->sev_data)
859 		return -ENODEV;
860 
861 	if (psp_dead)
862 		return -EBUSY;
863 
864 	sev = psp->sev_data;
865 
866 	buf_len = sev_cmd_buffer_len(cmd);
867 	if (WARN_ON_ONCE(!data != !buf_len))
868 		return -EINVAL;
869 
870 	/*
871 	 * Copy the incoming data to driver's scratch buffer as __pa() will not
872 	 * work for some memory, e.g. vmalloc'd addresses, and @data may not be
873 	 * physically contiguous.
874 	 */
875 	if (data) {
876 		/*
877 		 * Commands are generally issued one at a time and require the
878 		 * sev_cmd_mutex, but there could be recursive firmware requests
879 		 * due to SEV_CMD_SNP_PAGE_RECLAIM needing to be issued while
880 		 * preparing buffers for another command. This is the only known
881 		 * case of nesting in the current code, so exactly one
882 		 * additional command buffer is available for that purpose.
883 		 */
884 		if (!sev->cmd_buf_active) {
885 			cmd_buf = sev->cmd_buf;
886 			sev->cmd_buf_active = true;
887 		} else if (!sev->cmd_buf_backup_active) {
888 			cmd_buf = sev->cmd_buf_backup;
889 			sev->cmd_buf_backup_active = true;
890 		} else {
891 			dev_err(sev->dev,
892 				"SEV: too many firmware commands in progress, no command buffers available.\n");
893 			return -EBUSY;
894 		}
895 
896 		memcpy(cmd_buf, data, buf_len);
897 
898 		/*
899 		 * The behavior of the SEV-legacy commands is altered when the
900 		 * SNP firmware is in the INIT state.
901 		 */
902 		ret = snp_prep_cmd_buf(cmd, cmd_buf, desc_list);
903 		if (ret) {
904 			dev_err(sev->dev,
905 				"SEV: failed to prepare buffer for legacy command 0x%x. Error: %d\n",
906 				cmd, ret);
907 			return ret;
908 		}
909 	} else {
910 		cmd_buf = sev->cmd_buf;
911 	}
912 
913 	/* Get the physical address of the command buffer */
914 	phys_lsb = data ? lower_32_bits(__psp_pa(cmd_buf)) : 0;
915 	phys_msb = data ? upper_32_bits(__psp_pa(cmd_buf)) : 0;
916 
917 	dev_dbg(sev->dev, "sev command id %#x buffer 0x%08x%08x timeout %us\n",
918 		cmd, phys_msb, phys_lsb, psp_timeout);
919 
920 	print_hex_dump_debug("(in):  ", DUMP_PREFIX_OFFSET, 16, 2, data,
921 			     buf_len, false);
922 
923 	iowrite32(phys_lsb, sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
924 	iowrite32(phys_msb, sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
925 
926 	sev->int_rcvd = 0;
927 
928 	reg = FIELD_PREP(SEV_CMDRESP_CMD, cmd);
929 
930 	/*
931 	 * If invoked during panic handling, local interrupts are disabled so
932 	 * the PSP command completion interrupt can't be used.
933 	 * sev_wait_cmd_ioc() already checks for interrupts disabled and
934 	 * polls for PSP command completion.  Ensure we do not request an
935 	 * interrupt from the PSP if irqs disabled.
936 	 */
937 	if (!irqs_disabled())
938 		reg |= SEV_CMDRESP_IOC;
939 
940 	iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg);
941 
942 	/* wait for command completion */
943 	ret = sev_wait_cmd_ioc(sev, &reg, psp_timeout);
944 	if (ret) {
945 		if (psp_ret)
946 			*psp_ret = 0;
947 
948 		dev_err(sev->dev, "sev command %#x timed out, disabling PSP\n", cmd);
949 		psp_dead = true;
950 
951 		return ret;
952 	}
953 
954 	psp_timeout = psp_cmd_timeout;
955 
956 	if (psp_ret)
957 		*psp_ret = FIELD_GET(PSP_CMDRESP_STS, reg);
958 
959 	if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
960 		dev_dbg(sev->dev, "sev command %#x failed (%#010lx)\n",
961 			cmd, FIELD_GET(PSP_CMDRESP_STS, reg));
962 
963 		/*
964 		 * PSP firmware may report additional error information in the
965 		 * command buffer registers on error. Print contents of command
966 		 * buffer registers if they changed.
967 		 */
968 		cmdbuff_hi = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
969 		cmdbuff_lo = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
970 		if (cmdbuff_hi != phys_msb || cmdbuff_lo != phys_lsb) {
971 			dev_dbg(sev->dev, "Additional error information reported in cmdbuff:");
972 			dev_dbg(sev->dev, "  cmdbuff hi: %#010x\n", cmdbuff_hi);
973 			dev_dbg(sev->dev, "  cmdbuff lo: %#010x\n", cmdbuff_lo);
974 		}
975 		ret = -EIO;
976 	} else {
977 		ret = sev_write_init_ex_file_if_required(cmd);
978 	}
979 
980 	/*
981 	 * Copy potential output from the PSP back to data.  Do this even on
982 	 * failure in case the caller wants to glean something from the error.
983 	 */
984 	if (data) {
985 		int ret_reclaim;
986 		/*
987 		 * Restore the page state after the command completes.
988 		 */
989 		ret_reclaim = snp_reclaim_cmd_buf(cmd, cmd_buf);
990 		if (ret_reclaim) {
991 			dev_err(sev->dev,
992 				"SEV: failed to reclaim buffer for legacy command %#x. Error: %d\n",
993 				cmd, ret_reclaim);
994 			return ret_reclaim;
995 		}
996 
997 		memcpy(data, cmd_buf, buf_len);
998 
999 		if (sev->cmd_buf_backup_active)
1000 			sev->cmd_buf_backup_active = false;
1001 		else
1002 			sev->cmd_buf_active = false;
1003 
1004 		if (snp_unmap_cmd_buf_desc_list(desc_list))
1005 			return -EFAULT;
1006 	}
1007 
1008 	print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
1009 			     buf_len, false);
1010 
1011 	return ret;
1012 }
1013 
1014 int sev_do_cmd(int cmd, void *data, int *psp_ret)
1015 {
1016 	int rc;
1017 
1018 	mutex_lock(&sev_cmd_mutex);
1019 	rc = __sev_do_cmd_locked(cmd, data, psp_ret);
1020 	mutex_unlock(&sev_cmd_mutex);
1021 
1022 	return rc;
1023 }
1024 EXPORT_SYMBOL_GPL(sev_do_cmd);
1025 
1026 static int __sev_init_locked(int *error)
1027 {
1028 	struct sev_data_init data;
1029 
1030 	memset(&data, 0, sizeof(data));
1031 	if (sev_es_tmr) {
1032 		/*
1033 		 * Do not include the encryption mask on the physical
1034 		 * address of the TMR (firmware should clear it anyway).
1035 		 */
1036 		data.tmr_address = __pa(sev_es_tmr);
1037 
1038 		data.flags |= SEV_INIT_FLAGS_SEV_ES;
1039 		data.tmr_len = sev_es_tmr_size;
1040 	}
1041 
1042 	return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
1043 }
1044 
1045 static int __sev_init_ex_locked(int *error)
1046 {
1047 	struct sev_data_init_ex data;
1048 
1049 	memset(&data, 0, sizeof(data));
1050 	data.length = sizeof(data);
1051 	data.nv_address = __psp_pa(sev_init_ex_buffer);
1052 	data.nv_len = NV_LENGTH;
1053 
1054 	if (sev_es_tmr) {
1055 		/*
1056 		 * Do not include the encryption mask on the physical
1057 		 * address of the TMR (firmware should clear it anyway).
1058 		 */
1059 		data.tmr_address = __pa(sev_es_tmr);
1060 
1061 		data.flags |= SEV_INIT_FLAGS_SEV_ES;
1062 		data.tmr_len = sev_es_tmr_size;
1063 	}
1064 
1065 	return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
1066 }
1067 
1068 static inline int __sev_do_init_locked(int *psp_ret)
1069 {
1070 	if (sev_init_ex_buffer)
1071 		return __sev_init_ex_locked(psp_ret);
1072 	else
1073 		return __sev_init_locked(psp_ret);
1074 }
1075 
1076 /* Hypervisor Fixed pages API interface */
1077 static void snp_hv_fixed_pages_state_update(struct sev_device *sev,
1078 					    enum snp_hv_fixed_pages_state page_state)
1079 {
1080 	struct snp_hv_fixed_pages_entry *entry;
1081 
1082 	/* List is protected by sev_cmd_mutex */
1083 	lockdep_assert_held(&sev_cmd_mutex);
1084 
1085 	if (list_empty(&snp_hv_fixed_pages))
1086 		return;
1087 
1088 	list_for_each_entry(entry, &snp_hv_fixed_pages, list)
1089 		entry->page_state = page_state;
1090 }
1091 
1092 /*
1093  * Allocate HV_FIXED pages in 2MB aligned sizes to ensure the whole
1094  * 2MB pages are marked as HV_FIXED.
1095  */
1096 struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages)
1097 {
1098 	struct psp_device *psp_master = psp_get_master_device();
1099 	struct snp_hv_fixed_pages_entry *entry;
1100 	unsigned int order;
1101 	struct page *page;
1102 
1103 	if (!psp_master)
1104 		return NULL;
1105 
1106 	order = get_order(PMD_SIZE * num_2mb_pages);
1107 
1108 	/*
1109 	 * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1110 	 * also needs to be protected using the same mutex.
1111 	 */
1112 	guard(mutex)(&sev_cmd_mutex);
1113 
1114 	/*
1115 	 * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed
1116 	 * page state, fail if SNP is already initialized.
1117 	 */
1118 	if (psp_master->sev_data &&
1119 	    ((struct sev_device *)psp_master->sev_data)->snp_initialized)
1120 		return NULL;
1121 
1122 	/* Re-use freed pages that match the request */
1123 	list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1124 		/* Hypervisor fixed page allocator implements exact fit policy */
1125 		if (entry->order == order && entry->free) {
1126 			entry->free = false;
1127 			memset(page_address(entry->page), 0,
1128 			       (1 << entry->order) * PAGE_SIZE);
1129 			return entry->page;
1130 		}
1131 	}
1132 
1133 	page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
1134 	if (!page)
1135 		return NULL;
1136 
1137 	entry = kzalloc_obj(*entry);
1138 	if (!entry) {
1139 		__free_pages(page, order);
1140 		return NULL;
1141 	}
1142 
1143 	entry->page = page;
1144 	entry->order = order;
1145 	list_add_tail(&entry->list, &snp_hv_fixed_pages);
1146 
1147 	return page;
1148 }
1149 
1150 void snp_free_hv_fixed_pages(struct page *page)
1151 {
1152 	struct psp_device *psp_master = psp_get_master_device();
1153 	struct snp_hv_fixed_pages_entry *entry, *nentry;
1154 
1155 	if (!psp_master)
1156 		return;
1157 
1158 	/*
1159 	 * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1160 	 * also needs to be protected using the same mutex.
1161 	 */
1162 	guard(mutex)(&sev_cmd_mutex);
1163 
1164 	list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) {
1165 		if (entry->page != page)
1166 			continue;
1167 
1168 		/*
1169 		 * HV_FIXED page state cannot be changed until reboot
1170 		 * and they cannot be used by an SNP guest, so they cannot
1171 		 * be returned back to the page allocator.
1172 		 * Mark the pages as free internally to allow possible re-use.
1173 		 */
1174 		if (entry->page_state == HV_FIXED) {
1175 			entry->free = true;
1176 		} else {
1177 			__free_pages(page, entry->order);
1178 			list_del(&entry->list);
1179 			kfree(entry);
1180 		}
1181 		return;
1182 	}
1183 }
1184 
1185 static void snp_add_hv_fixed_pages(struct sev_device *sev, struct sev_data_range_list *range_list)
1186 {
1187 	struct snp_hv_fixed_pages_entry *entry;
1188 	struct sev_data_range *range;
1189 	int num_elements;
1190 
1191 	lockdep_assert_held(&sev_cmd_mutex);
1192 
1193 	if (list_empty(&snp_hv_fixed_pages))
1194 		return;
1195 
1196 	num_elements = list_count_nodes(&snp_hv_fixed_pages) +
1197 		       range_list->num_elements;
1198 
1199 	/*
1200 	 * Ensure the list of HV_FIXED pages that will be passed to firmware
1201 	 * do not exceed the page-sized argument buffer.
1202 	 */
1203 	if (num_elements * sizeof(*range) + sizeof(*range_list) > PAGE_SIZE) {
1204 		dev_warn(sev->dev, "Additional HV_Fixed pages cannot be accommodated, omitting\n");
1205 		return;
1206 	}
1207 
1208 	range = &range_list->ranges[range_list->num_elements];
1209 	list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1210 		range->base = page_to_pfn(entry->page) << PAGE_SHIFT;
1211 		range->page_count = 1 << entry->order;
1212 		range++;
1213 	}
1214 	range_list->num_elements = num_elements;
1215 }
1216 
1217 static void snp_leak_hv_fixed_pages(void)
1218 {
1219 	struct snp_hv_fixed_pages_entry *entry, *nentry;
1220 
1221 	/* List is protected by sev_cmd_mutex */
1222 	lockdep_assert_held(&sev_cmd_mutex);
1223 
1224 	if (list_empty(&snp_hv_fixed_pages))
1225 		return;
1226 
1227 	list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) {
1228 		if (entry->free && entry->page_state != HV_FIXED)
1229 			__free_pages(entry->page, entry->order);
1230 		else
1231 			__snp_leak_pages(page_to_pfn(entry->page),
1232 					 1 << entry->order, false);
1233 
1234 		list_del(&entry->list);
1235 		kfree(entry);
1236 	}
1237 }
1238 
1239 bool sev_is_snp_ciphertext_hiding_supported(void)
1240 {
1241 	struct psp_device *psp = psp_master;
1242 	struct sev_device *sev;
1243 
1244 	if (!psp || !psp->sev_data)
1245 		return false;
1246 
1247 	sev = psp->sev_data;
1248 
1249 	/*
1250 	 * Feature information indicates if CipherTextHiding feature is
1251 	 * supported by the SEV firmware and additionally platform status
1252 	 * indicates if CipherTextHiding feature is enabled in the
1253 	 * Platform BIOS.
1254 	 */
1255 	return ((sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) &&
1256 		 sev->snp_plat_status.ciphertext_hiding_cap);
1257 }
1258 EXPORT_SYMBOL_GPL(sev_is_snp_ciphertext_hiding_supported);
1259 
1260 static int snp_get_platform_data(struct sev_device *sev, int *error)
1261 {
1262 	struct sev_data_snp_feature_info snp_feat_info;
1263 	struct snp_feature_info *feat_info;
1264 	struct sev_data_snp_addr buf;
1265 	struct page *page;
1266 	int rc;
1267 
1268 	/*
1269 	 * This function is expected to be called before SNP is
1270 	 * initialized.
1271 	 */
1272 	if (sev->snp_initialized)
1273 		return -EINVAL;
1274 
1275 	buf.address = __psp_pa(&sev->snp_plat_status);
1276 	rc = sev_do_cmd(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error);
1277 	if (rc) {
1278 		dev_err(sev->dev, "SNP PLATFORM_STATUS command failed, ret = %d, error = %#x\n",
1279 			rc, *error);
1280 		return rc;
1281 	}
1282 
1283 	sev->api_major = sev->snp_plat_status.api_major;
1284 	sev->api_minor = sev->snp_plat_status.api_minor;
1285 	sev->build = sev->snp_plat_status.build_id;
1286 
1287 	/*
1288 	 * Do feature discovery of the currently loaded firmware,
1289 	 * and cache feature information from CPUID 0x8000_0024,
1290 	 * sub-function 0.
1291 	 */
1292 	if (!sev->snp_plat_status.feature_info)
1293 		return 0;
1294 
1295 	/*
1296 	 * Use dynamically allocated structure for the SNP_FEATURE_INFO
1297 	 * command to ensure structure is 8-byte aligned, and does not
1298 	 * cross a page boundary.
1299 	 */
1300 	page = alloc_page(GFP_KERNEL);
1301 	if (!page)
1302 		return -ENOMEM;
1303 
1304 	feat_info = page_address(page);
1305 	snp_feat_info.length = sizeof(snp_feat_info);
1306 	snp_feat_info.ecx_in = 0;
1307 	snp_feat_info.feature_info_paddr = __psp_pa(feat_info);
1308 
1309 	rc = sev_do_cmd(SEV_CMD_SNP_FEATURE_INFO, &snp_feat_info, error);
1310 	if (!rc)
1311 		sev->snp_feat_info_0 = *feat_info;
1312 	else
1313 		dev_err(sev->dev, "SNP FEATURE_INFO command failed, ret = %d, error = %#x\n",
1314 			rc, *error);
1315 
1316 	__free_page(page);
1317 
1318 	return rc;
1319 }
1320 
1321 static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg)
1322 {
1323 	struct sev_data_range_list *range_list = arg;
1324 	struct sev_data_range *range = &range_list->ranges[range_list->num_elements];
1325 	size_t size;
1326 
1327 	/*
1328 	 * Ensure the list of HV_FIXED pages passed to the firmware including
1329 	 * the one about to be written to do not exceed the page-sized argument
1330 	 * buffer.
1331 	 */
1332 	if (((range_list->num_elements + 1) * sizeof(struct sev_data_range) +
1333 	     sizeof(struct sev_data_range_list)) > PAGE_SIZE)
1334 		return -E2BIG;
1335 
1336 	switch (rs->desc) {
1337 	case E820_TYPE_RESERVED:
1338 	case E820_TYPE_PMEM:
1339 	case E820_TYPE_ACPI:
1340 		range->base = rs->start & PAGE_MASK;
1341 		size = PAGE_ALIGN((rs->end + 1) - rs->start);
1342 		range->page_count = size >> PAGE_SHIFT;
1343 		range_list->num_elements++;
1344 		break;
1345 	default:
1346 		break;
1347 	}
1348 
1349 	return 0;
1350 }
1351 
1352 #ifdef CONFIG_SYSFS
1353 static int snp_verify_mitigation(u16 command, u64 vector,
1354 				 struct sev_data_snp_verify_mitigation_dst *dst)
1355 {
1356 	struct sev_data_snp_verify_mitigation_dst *mit_dst = NULL;
1357 	struct sev_data_snp_verify_mitigation data = {0};
1358 	struct sev_device *sev = psp_master->sev_data;
1359 	int ret, error = 0;
1360 
1361 	mit_dst = snp_alloc_firmware_page(GFP_KERNEL | __GFP_ZERO);
1362 	if (!mit_dst)
1363 		return -ENOMEM;
1364 
1365 	data.length = sizeof(data);
1366 	data.subcommand = command;
1367 	data.vector = vector;
1368 	data.dst_paddr = __psp_pa(mit_dst);
1369 	data.dst_paddr_en = true;
1370 
1371 	ret = sev_do_cmd(SEV_CMD_SNP_VERIFY_MITIGATION, &data, &error);
1372 	if (!ret)
1373 		memcpy(dst, mit_dst, sizeof(*mit_dst));
1374 	else
1375 		dev_err(sev->dev, "SNP_VERIFY_MITIGATION command failed, ret = %d, error = %#x\n",
1376 			ret, error);
1377 
1378 	snp_free_firmware_page(mit_dst);
1379 
1380 	return ret;
1381 }
1382 
1383 static ssize_t supported_mitigations_show(struct kobject *kobj,
1384 					  struct kobj_attribute *attr, char *buf)
1385 {
1386 	struct sev_data_snp_verify_mitigation_dst dst;
1387 	int ret;
1388 
1389 	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst);
1390 	if (ret)
1391 		return ret;
1392 
1393 	return sysfs_emit(buf, "0x%llx\n", dst.mit_supported_vector);
1394 }
1395 
1396 static struct kobj_attribute supported_attr =
1397 		__ATTR_RO_MODE(supported_mitigations, 0400);
1398 
1399 static ssize_t verified_mitigations_show(struct kobject *kobj,
1400 					 struct kobj_attribute *attr, char *buf)
1401 {
1402 	struct sev_data_snp_verify_mitigation_dst dst;
1403 	int ret;
1404 
1405 	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst);
1406 	if (ret)
1407 		return ret;
1408 
1409 	return sysfs_emit(buf, "0x%llx\n", dst.mit_verified_vector);
1410 }
1411 
1412 static ssize_t verified_mitigations_store(struct kobject *kobj,
1413 					  struct kobj_attribute *attr,
1414 					  const char *buf, size_t count)
1415 {
1416 	struct sev_data_snp_verify_mitigation_dst dst;
1417 	struct sev_device *sev = psp_master->sev_data;
1418 	u64 vector;
1419 	int ret;
1420 
1421 	ret = kstrtoull(buf, 0, &vector);
1422 	if (ret)
1423 		return ret;
1424 
1425 	/*
1426 	 * The firmware verifies a single mitigation per call. Reject vectors
1427 	 * with more than one bit set early to avoid a guaranteed-to-fail call
1428 	 */
1429 	if (hweight64(vector) != 1)
1430 		return -EINVAL;
1431 
1432 	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_VERIFY, vector, &dst);
1433 	if (ret)
1434 		return ret;
1435 
1436 	if (dst.mit_failure_status) {
1437 		dev_err(sev->dev, "Verify Mitigation - failure status: 0x%x\n",
1438 			dst.mit_failure_status);
1439 		return -EINVAL;
1440 	}
1441 
1442 	return count;
1443 }
1444 
1445 static struct kobj_attribute verified_attr =
1446 		__ATTR_RW_MODE(verified_mitigations, 0600);
1447 
1448 static struct attribute *mitigation_attrs[] = {
1449 	&supported_attr.attr,
1450 	&verified_attr.attr,
1451 	NULL
1452 };
1453 
1454 static const struct attribute_group mit_attr_group = {
1455 	.attrs = mitigation_attrs,
1456 };
1457 
1458 static void sev_snp_register_verify_mitigation(struct sev_device *sev)
1459 {
1460 	int rc;
1461 
1462 	if (!(sev->snp_feat_info_0.ecx & SNP_VERIFY_MITIGATION_SUPPORTED) ||
1463 	    sev->verify_mit)
1464 		return;
1465 
1466 	if (!sev->sev_kobj) {
1467 		sev->sev_kobj = kobject_create_and_add("sev", firmware_kobj);
1468 		if (!sev->sev_kobj)
1469 			return;
1470 	}
1471 
1472 	sev->verify_mit = kobject_create_and_add("vulnerabilities", sev->sev_kobj);
1473 	if (!sev->verify_mit)
1474 		goto err_sev_kobj;
1475 
1476 	rc = sysfs_create_group(sev->verify_mit, &mit_attr_group);
1477 	if (rc)
1478 		goto err_verify_mit;
1479 
1480 	return;
1481 
1482 err_verify_mit:
1483 	kobject_put(sev->verify_mit);
1484 	sev->verify_mit = NULL;
1485 err_sev_kobj:
1486 	kobject_put(sev->sev_kobj);
1487 	sev->sev_kobj = NULL;
1488 }
1489 
1490 static void sev_snp_unregister_verify_mitigation(struct sev_device *sev)
1491 {
1492 	if (sev->verify_mit) {
1493 		sysfs_remove_group(sev->verify_mit, &mit_attr_group);
1494 		kobject_put(sev->verify_mit);
1495 		sev->verify_mit = NULL;
1496 	}
1497 
1498 	if (sev->sev_kobj) {
1499 		kobject_put(sev->sev_kobj);
1500 		sev->sev_kobj = NULL;
1501 	}
1502 }
1503 #else	// CONFIG_SYSFS
1504 static void sev_snp_register_verify_mitigation(struct sev_device *sev) { }
1505 static void sev_snp_unregister_verify_mitigation(struct sev_device *sev) { }
1506 #endif	// CONFIG_SYSFS
1507 
1508 static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
1509 {
1510 	struct sev_data_range_list *snp_range_list __free(kfree) = NULL;
1511 	struct psp_device *psp = psp_master;
1512 	struct sev_data_snp_init_ex data = {};
1513 	struct sev_device *sev;
1514 	void *arg = &data;
1515 	int cmd, rc = 0;
1516 
1517 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1518 		return -ENODEV;
1519 
1520 	sev = psp->sev_data;
1521 
1522 	if (sev->snp_initialized)
1523 		return 0;
1524 
1525 	if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) {
1526 		dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n",
1527 			SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR);
1528 		return -EOPNOTSUPP;
1529 	}
1530 
1531 	rc = snp_prepare();
1532 	if (rc)
1533 		return rc;
1534 
1535 	/*
1536 	 * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
1537 	 * of system physical address ranges to convert into HV-fixed page
1538 	 * states during the RMP initialization.  For instance, the memory that
1539 	 * UEFI reserves should be included in the that list. This allows system
1540 	 * components that occasionally write to memory (e.g. logging to UEFI
1541 	 * reserved regions) to not fail due to RMP initialization and SNP
1542 	 * enablement.
1543 	 *
1544 	 */
1545 	if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) {
1546 		bool tio_supp = !!(sev->snp_feat_info_0.ebx & SNP_SEV_TIO_SUPPORTED);
1547 
1548 		/*
1549 		 * Firmware checks that the pages containing the ranges enumerated
1550 		 * in the RANGES structure are either in the default page state or in the
1551 		 * firmware page state.
1552 		 */
1553 		snp_range_list = kzalloc(PAGE_SIZE, GFP_KERNEL);
1554 		if (!snp_range_list) {
1555 			dev_err(sev->dev,
1556 				"SEV: SNP_INIT_EX range list memory allocation failed\n");
1557 			return -ENOMEM;
1558 		}
1559 
1560 		/*
1561 		 * Retrieve all reserved memory regions from the e820 memory map
1562 		 * to be setup as HV-fixed pages.
1563 		 */
1564 		rc = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM, 0, ~0,
1565 					 snp_range_list, snp_filter_reserved_mem_regions);
1566 		if (rc) {
1567 			dev_err(sev->dev,
1568 				"SEV: SNP_INIT_EX walk_iomem_res_desc failed rc = %d\n", rc);
1569 			return rc;
1570 		}
1571 
1572 		/*
1573 		 * Add HV_Fixed pages from other PSP sub-devices, such as SFS to the
1574 		 * HV_Fixed page list.
1575 		 */
1576 		snp_add_hv_fixed_pages(sev, snp_range_list);
1577 
1578 		if (max_snp_asid) {
1579 			data.ciphertext_hiding_en = 1;
1580 			data.max_snp_asid = max_snp_asid;
1581 		}
1582 
1583 		data.init_rmp = 1;
1584 		data.list_paddr_en = 1;
1585 		data.list_paddr = __psp_pa(snp_range_list);
1586 
1587 		data.tio_en = tio_supp && sev_tio_enabled && amd_iommu_sev_tio_supported();
1588 
1589 		/*
1590 		 * When psp_init_on_probe is disabled, the userspace calling
1591 		 * SEV ioctl can inadvertently shut down SNP and SEV-TIO causing
1592 		 * unexpected state loss.
1593 		 */
1594 		if (data.tio_en && !psp_init_on_probe)
1595 			dev_warn(sev->dev, "SEV-TIO as incompatible with psp_init_on_probe=0\n");
1596 
1597 		cmd = SEV_CMD_SNP_INIT_EX;
1598 	} else {
1599 		cmd = SEV_CMD_SNP_INIT;
1600 		arg = NULL;
1601 	}
1602 
1603 	/*
1604 	 * The following sequence must be issued before launching the first SNP
1605 	 * guest to ensure all dirty cache lines are flushed, including from
1606 	 * updates to the RMP table itself via the RMPUPDATE instruction:
1607 	 *
1608 	 * - WBINVD on all running CPUs
1609 	 * - SEV_CMD_SNP_INIT[_EX] firmware command
1610 	 * - WBINVD on all running CPUs
1611 	 * - SEV_CMD_SNP_DF_FLUSH firmware command
1612 	 */
1613 	wbinvd_on_all_cpus();
1614 
1615 	rc = __sev_do_cmd_locked(cmd, arg, error);
1616 	if (rc) {
1617 		dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n",
1618 			cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT",
1619 			rc, *error);
1620 		return rc;
1621 	}
1622 
1623 	/* Prepare for first SNP guest launch after INIT. */
1624 	wbinvd_on_all_cpus();
1625 	rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error);
1626 	if (rc) {
1627 		dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n",
1628 			rc, *error);
1629 		return rc;
1630 	}
1631 
1632 	snp_hv_fixed_pages_state_update(sev, HV_FIXED);
1633 	sev->snp_initialized = true;
1634 	dev_dbg(sev->dev, "SEV-SNP firmware initialized, SEV-TIO is %s\n",
1635 		data.tio_en ? "enabled" : "disabled");
1636 
1637 	dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major,
1638 		 sev->api_minor, sev->build);
1639 
1640 	atomic_notifier_chain_register(&panic_notifier_list,
1641 				       &snp_panic_notifier);
1642 
1643 	if (data.tio_en) {
1644 		struct page *page;
1645 
1646 		/*
1647 		 * This executes with the sev_cmd_mutex held so down the stack
1648 		 * snp_reclaim_pages(locked=false) might be needed (which is extremely
1649 		 * unlikely) but will cause a deadlock.
1650 		 * Instead of exporting __snp_alloc_firmware_pages(), allocate a page
1651 		 * for this one call here.
1652 		 */
1653 		page = __snp_alloc_firmware_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1654 						  0, true);
1655 		if (page) {
1656 			void *tio_status = page_address(page);
1657 
1658 			sev_tsm_init_locked(sev, tio_status);
1659 
1660 			__snp_free_firmware_pages(page, 0, true);
1661 		}
1662 	}
1663 
1664 	sev_es_tmr_size = SNP_TMR_SIZE;
1665 
1666 	snp_enable_rmpopt();
1667 
1668 	return 0;
1669 }
1670 
1671 static void __sev_platform_init_handle_tmr(struct sev_device *sev)
1672 {
1673 	if (sev_es_tmr)
1674 		return;
1675 
1676 	/* Obtain the TMR memory area for SEV-ES use */
1677 	sev_es_tmr = sev_fw_alloc(sev_es_tmr_size);
1678 	if (sev_es_tmr) {
1679 		/* Must flush the cache before giving it to the firmware */
1680 		if (!sev->snp_initialized)
1681 			clflush_cache_range(sev_es_tmr, sev_es_tmr_size);
1682 	} else {
1683 			dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n");
1684 	}
1685 }
1686 
1687 /*
1688  * If an init_ex_path is provided allocate a buffer for the file and
1689  * read in the contents. Additionally, if SNP is initialized, convert
1690  * the buffer pages to firmware pages.
1691  */
1692 static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev)
1693 {
1694 	struct page *page;
1695 	int rc;
1696 
1697 	if (!init_ex_path)
1698 		return 0;
1699 
1700 	if (sev_init_ex_buffer)
1701 		return 0;
1702 
1703 	page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(NV_LENGTH));
1704 	if (!page) {
1705 		dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n");
1706 		return -ENOMEM;
1707 	}
1708 
1709 	sev_init_ex_buffer = page_address(page);
1710 
1711 	rc = sev_read_init_ex_file();
1712 	if (rc)
1713 		goto err_free;
1714 
1715 	/* If SEV-SNP is initialized, transition to firmware page. */
1716 	if (sev->snp_initialized) {
1717 		unsigned long npages;
1718 
1719 		npages = 1UL << get_order(NV_LENGTH);
1720 		if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, true)) {
1721 			dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n");
1722 			rc = -ENOMEM;
1723 			/*
1724 			 * Pages can be in an inconsistent state, don't release them back to the
1725 			 * system.
1726 			 */
1727 			goto err_reset;
1728 		}
1729 	}
1730 
1731 	return 0;
1732 
1733 err_free:
1734 	__free_pages(page, get_order(NV_LENGTH));
1735 err_reset:
1736 	sev_init_ex_buffer = NULL;
1737 	return rc;
1738 }
1739 
1740 static int __sev_platform_init_locked(int *error)
1741 {
1742 	int rc, psp_ret, dfflush_error;
1743 	struct sev_device *sev;
1744 
1745 	psp_ret = dfflush_error = SEV_RET_NO_FW_CALL;
1746 
1747 	if (!psp_master || !psp_master->sev_data)
1748 		return -ENODEV;
1749 
1750 	sev = psp_master->sev_data;
1751 
1752 	if (sev->sev_plat_status.state == SEV_STATE_INIT)
1753 		return 0;
1754 
1755 	__sev_platform_init_handle_tmr(sev);
1756 
1757 	rc = __sev_platform_init_handle_init_ex_path(sev);
1758 	if (rc)
1759 		return rc;
1760 
1761 	rc = __sev_do_init_locked(&psp_ret);
1762 	if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
1763 		/*
1764 		 * Initialization command returned an integrity check failure
1765 		 * status code, meaning that firmware load and validation of SEV
1766 		 * related persistent data has failed. Retrying the
1767 		 * initialization function should succeed by replacing the state
1768 		 * with a reset state.
1769 		 */
1770 		dev_err(sev->dev,
1771 "SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state.");
1772 		rc = __sev_do_init_locked(&psp_ret);
1773 	}
1774 
1775 	if (error)
1776 		*error = psp_ret;
1777 
1778 	if (rc) {
1779 		dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n",
1780 			sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc);
1781 		return rc;
1782 	}
1783 
1784 	sev->sev_plat_status.state = SEV_STATE_INIT;
1785 
1786 	/* Prepare for first SEV guest launch after INIT */
1787 	wbinvd_on_all_cpus();
1788 	rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, &dfflush_error);
1789 	if (rc) {
1790 		dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n",
1791 			dfflush_error, rc);
1792 		return rc;
1793 	}
1794 
1795 	dev_dbg(sev->dev, "SEV firmware initialized\n");
1796 
1797 	dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
1798 		 sev->api_minor, sev->build);
1799 
1800 	return 0;
1801 }
1802 
1803 static int _sev_platform_init_locked(struct sev_platform_init_args *args)
1804 {
1805 	struct sev_device *sev;
1806 	int rc;
1807 
1808 	if (!psp_master || !psp_master->sev_data)
1809 		return -ENODEV;
1810 
1811 	/*
1812 	 * Skip SNP/SEV initialization under a kdump kernel as SEV/SNP
1813 	 * may already be initialized in the previous kernel. Since no
1814 	 * SNP/SEV guests are run under a kdump kernel, there is no
1815 	 * need to initialize SNP or SEV during kdump boot.
1816 	 */
1817 	if (is_kdump_kernel())
1818 		return 0;
1819 
1820 	sev = psp_master->sev_data;
1821 
1822 	if (sev->sev_plat_status.state == SEV_STATE_INIT)
1823 		return 0;
1824 
1825 	rc = __sev_snp_init_locked(&args->error, args->max_snp_asid);
1826 	if (rc && rc != -ENODEV)
1827 		return rc;
1828 
1829 	/* Defer legacy SEV/SEV-ES support if allowed by caller/module. */
1830 	if (args->probe && !psp_init_on_probe)
1831 		return 0;
1832 
1833 	return __sev_platform_init_locked(&args->error);
1834 }
1835 
1836 int sev_platform_init(struct sev_platform_init_args *args)
1837 {
1838 	int rc;
1839 
1840 	mutex_lock(&sev_cmd_mutex);
1841 	rc = _sev_platform_init_locked(args);
1842 	mutex_unlock(&sev_cmd_mutex);
1843 
1844 	/*
1845 	 * Register the sysfs interface outside the sev_cmd_mutex. The
1846 	 * _show()/_store() handlers issue SEV commands that acquire the
1847 	 * sev_cmd_mutex, so creating (and on the shutdown path, removing) the
1848 	 * sysfs group must stay outside that lock. sysfs provides its own
1849 	 * synchronization between group creation/removal and concurrent
1850 	 * attribute access.
1851 	 */
1852 	if (!rc)
1853 		sev_snp_register_verify_mitigation(psp_master->sev_data);
1854 
1855 	return rc;
1856 }
1857 EXPORT_SYMBOL_GPL(sev_platform_init);
1858 
1859 static int __sev_platform_shutdown_locked(int *error)
1860 {
1861 	struct psp_device *psp = psp_master;
1862 	struct sev_device *sev;
1863 	int ret;
1864 
1865 	if (!psp || !psp->sev_data)
1866 		return 0;
1867 
1868 	sev = psp->sev_data;
1869 
1870 	if (sev->sev_plat_status.state == SEV_STATE_UNINIT)
1871 		return 0;
1872 
1873 	ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error);
1874 	if (ret) {
1875 		dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n",
1876 			*error, ret);
1877 		return ret;
1878 	}
1879 
1880 	sev->sev_plat_status.state = SEV_STATE_UNINIT;
1881 	dev_dbg(sev->dev, "SEV firmware shutdown\n");
1882 
1883 	return ret;
1884 }
1885 
1886 static int sev_get_platform_state(int *state, int *error)
1887 {
1888 	struct sev_user_data_status data;
1889 	int rc;
1890 
1891 	rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, error);
1892 	if (rc)
1893 		return rc;
1894 
1895 	*state = data.state;
1896 	return rc;
1897 }
1898 
1899 static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
1900 {
1901 	int rc;
1902 
1903 	rc = __sev_platform_init_locked(&argp->error);
1904 	if (rc)
1905 		return rc;
1906 
1907 	*shutdown_required = true;
1908 
1909 	return 0;
1910 }
1911 
1912 static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable)
1913 {
1914 	int state, rc;
1915 
1916 	if (!writable)
1917 		return -EPERM;
1918 
1919 	mutex_lock(&sev_cmd_mutex);
1920 
1921 	/*
1922 	 * The SEV spec requires that FACTORY_RESET must be issued in
1923 	 * UNINIT state. Before we go further lets check if any guest is
1924 	 * active.
1925 	 *
1926 	 * If FW is in WORKING state then deny the request otherwise issue
1927 	 * SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET.
1928 	 *
1929 	 */
1930 	rc = sev_get_platform_state(&state, &argp->error);
1931 	if (rc)
1932 		goto out_unlock;
1933 
1934 	if (state == SEV_STATE_WORKING) {
1935 		rc = -EBUSY;
1936 		goto out_unlock;
1937 	}
1938 
1939 	if (state == SEV_STATE_INIT) {
1940 		rc = __sev_platform_shutdown_locked(&argp->error);
1941 		if (rc)
1942 			goto out_unlock;
1943 	}
1944 
1945 	rc = __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error);
1946 
1947 out_unlock:
1948 	mutex_unlock(&sev_cmd_mutex);
1949 	return rc;
1950 }
1951 
1952 static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp)
1953 {
1954 	struct sev_user_data_status data;
1955 	int ret;
1956 
1957 	memset(&data, 0, sizeof(data));
1958 
1959 	mutex_lock(&sev_cmd_mutex);
1960 	ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error);
1961 	mutex_unlock(&sev_cmd_mutex);
1962 	if (ret)
1963 		return ret;
1964 
1965 	if (copy_to_user((void __user *)argp->data, &data, sizeof(data)))
1966 		ret = -EFAULT;
1967 
1968 	return ret;
1969 }
1970 
1971 static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable)
1972 {
1973 	struct sev_device *sev = psp_master->sev_data;
1974 	bool shutdown_required = false;
1975 	int rc;
1976 
1977 	if (!writable)
1978 		return -EPERM;
1979 
1980 	mutex_lock(&sev_cmd_mutex);
1981 
1982 	if (sev->sev_plat_status.state == SEV_STATE_UNINIT) {
1983 		rc = sev_move_to_init_state(argp, &shutdown_required);
1984 		if (rc)
1985 			goto out_unlock;
1986 	}
1987 
1988 	rc = __sev_do_cmd_locked(cmd, NULL, &argp->error);
1989 
1990 	if (shutdown_required)
1991 		__sev_firmware_shutdown(sev, false);
1992 
1993 out_unlock:
1994 	mutex_unlock(&sev_cmd_mutex);
1995 
1996 	return rc;
1997 }
1998 
1999 static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable)
2000 {
2001 	struct sev_device *sev = psp_master->sev_data;
2002 	struct sev_user_data_pek_csr input;
2003 	bool shutdown_required = false;
2004 	struct sev_data_pek_csr data;
2005 	void __user *input_address;
2006 	void *blob = NULL;
2007 	int ret;
2008 
2009 	if (!writable)
2010 		return -EPERM;
2011 
2012 	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2013 		return -EFAULT;
2014 
2015 	memset(&data, 0, sizeof(data));
2016 
2017 	/* userspace wants to query CSR length */
2018 	if (!input.address || !input.length)
2019 		goto cmd;
2020 
2021 	/* allocate a physically contiguous buffer to store the CSR blob */
2022 	input_address = (void __user *)input.address;
2023 	if (input.length > SEV_FW_BLOB_MAX_SIZE)
2024 		return -EFAULT;
2025 
2026 	blob = kzalloc(input.length, GFP_KERNEL);
2027 	if (!blob)
2028 		return -ENOMEM;
2029 
2030 	data.address = __psp_pa(blob);
2031 	data.len = input.length;
2032 
2033 cmd:
2034 	mutex_lock(&sev_cmd_mutex);
2035 
2036 	if (sev->sev_plat_status.state == SEV_STATE_UNINIT) {
2037 		ret = sev_move_to_init_state(argp, &shutdown_required);
2038 		if (ret) {
2039 			mutex_unlock(&sev_cmd_mutex);
2040 			/* No shutdown is needed if init fails */
2041 			goto e_free_blob;
2042 		}
2043 	}
2044 
2045 	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error);
2046 
2047 	if (shutdown_required)
2048 		__sev_firmware_shutdown(sev, false);
2049 
2050 	mutex_unlock(&sev_cmd_mutex);
2051 
2052 	/*
2053 	 * Firmware will returns the length of the CSR blob (either the minimum
2054 	 * required length or the actual length written), return it to the user.
2055 	 */
2056 	input.length = data.len;
2057 
2058 	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2059 		ret = -EFAULT;
2060 		goto e_free_blob;
2061 	}
2062 
2063 	if (ret || WARN_ON_ONCE(argp->error))
2064 		goto e_free_blob;
2065 
2066 	if (blob) {
2067 		if (copy_to_user(input_address, blob, input.length))
2068 			ret = -EFAULT;
2069 	}
2070 
2071 e_free_blob:
2072 	kfree(blob);
2073 	return ret;
2074 }
2075 
2076 void *psp_copy_user_blob(u64 uaddr, u32 len)
2077 {
2078 	if (!uaddr || !len)
2079 		return ERR_PTR(-EINVAL);
2080 
2081 	/* verify that blob length does not exceed our limit */
2082 	if (len > SEV_FW_BLOB_MAX_SIZE)
2083 		return ERR_PTR(-EINVAL);
2084 
2085 	return memdup_user((void __user *)uaddr, len);
2086 }
2087 EXPORT_SYMBOL_GPL(psp_copy_user_blob);
2088 
2089 static int sev_get_api_version(void)
2090 {
2091 	struct sev_device *sev = psp_master->sev_data;
2092 	struct sev_user_data_status status;
2093 	int error = 0, ret;
2094 
2095 	/*
2096 	 * Cache SNP platform status and SNP feature information
2097 	 * if SNP is available.
2098 	 */
2099 	if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) {
2100 		ret = snp_get_platform_data(sev, &error);
2101 		if (ret)
2102 			return 1;
2103 	}
2104 
2105 	ret = sev_platform_status(&status, &error);
2106 	if (ret) {
2107 		dev_err(sev->dev,
2108 			"SEV: failed to get status. Error: %#x\n", error);
2109 		return 1;
2110 	}
2111 
2112 	/* Cache SEV platform status */
2113 	sev->sev_plat_status = status;
2114 
2115 	sev->api_major = status.api_major;
2116 	sev->api_minor = status.api_minor;
2117 	sev->build = status.build;
2118 
2119 	return 0;
2120 }
2121 
2122 static int sev_get_firmware(struct device *dev,
2123 			    const struct firmware **firmware)
2124 {
2125 	char fw_name_specific[SEV_FW_NAME_SIZE];
2126 	char fw_name_subset[SEV_FW_NAME_SIZE];
2127 
2128 	snprintf(fw_name_specific, sizeof(fw_name_specific),
2129 		 "amd/amd_sev_fam%.2xh_model%.2xh.sbin",
2130 		 boot_cpu_data.x86, boot_cpu_data.x86_model);
2131 
2132 	snprintf(fw_name_subset, sizeof(fw_name_subset),
2133 		 "amd/amd_sev_fam%.2xh_model%.1xxh.sbin",
2134 		 boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4);
2135 
2136 	/* Check for SEV FW for a particular model.
2137 	 * Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h
2138 	 *
2139 	 * or
2140 	 *
2141 	 * Check for SEV FW common to a subset of models.
2142 	 * Ex. amd_sev_fam17h_model0xh.sbin for
2143 	 *     Family 17h Model 00h -- Family 17h Model 0Fh
2144 	 *
2145 	 * or
2146 	 *
2147 	 * Fall-back to using generic name: sev.fw
2148 	 */
2149 	if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) ||
2150 	    (firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) ||
2151 	    (firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0))
2152 		return 0;
2153 
2154 	return -ENOENT;
2155 }
2156 
2157 /* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
2158 static int sev_update_firmware(struct device *dev)
2159 {
2160 	struct sev_data_download_firmware data;
2161 	const struct firmware *firmware;
2162 	int ret, error, order;
2163 	struct page *p;
2164 	void *fw_blob;
2165 
2166 	if (!sev_version_greater_or_equal(0, 15)) {
2167 		dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n");
2168 		return -1;
2169 	}
2170 
2171 	if (sev_get_firmware(dev, &firmware) == -ENOENT) {
2172 		dev_dbg(dev, "No SEV firmware file present\n");
2173 		return -1;
2174 	}
2175 
2176 	order = get_order(firmware->size);
2177 	p = alloc_pages(GFP_KERNEL, order);
2178 	if (!p) {
2179 		ret = -1;
2180 		goto fw_err;
2181 	}
2182 
2183 	/*
2184 	 * Copy firmware data to a kernel allocated contiguous
2185 	 * memory region.
2186 	 */
2187 	fw_blob = page_address(p);
2188 	memcpy(fw_blob, firmware->data, firmware->size);
2189 
2190 	data.address = __psp_pa(fw_blob);
2191 	data.len = firmware->size;
2192 
2193 	ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error);
2194 
2195 	/*
2196 	 * A quirk for fixing the committed TCB version, when upgrading from
2197 	 * earlier firmware version than 1.50.
2198 	 */
2199 	if (!ret && !sev_version_greater_or_equal(1, 50))
2200 		ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error);
2201 
2202 	if (ret)
2203 		dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error);
2204 
2205 	__free_pages(p, order);
2206 
2207 fw_err:
2208 	release_firmware(firmware);
2209 
2210 	return ret;
2211 }
2212 
2213 static int __sev_snp_shutdown_locked(int *error, bool panic)
2214 {
2215 	struct psp_device *psp = psp_master;
2216 	struct sev_device *sev;
2217 	struct sev_data_snp_shutdown_ex data;
2218 	int ret;
2219 
2220 	if (!psp || !psp->sev_data)
2221 		return 0;
2222 
2223 	sev = psp->sev_data;
2224 
2225 	if (!sev->snp_initialized)
2226 		return 0;
2227 
2228 	memset(&data, 0, sizeof(data));
2229 	data.len = sizeof(data);
2230 	data.iommu_snp_shutdown = 1;
2231 	if (sev->snp_feat_info_0.ecx & SNP_X86_SHUTDOWN_SUPPORTED)
2232 		data.x86_snp_shutdown = 1;
2233 
2234 	/*
2235 	 * If invoked during panic handling, local interrupts are disabled
2236 	 * and all CPUs are stopped, so wbinvd_on_all_cpus() can't be called.
2237 	 * In that case, a wbinvd() is done on remote CPUs via the NMI
2238 	 * callback, so only a local wbinvd() is needed here.
2239 	 */
2240 	if (!panic)
2241 		wbinvd_on_all_cpus();
2242 	else
2243 		wbinvd();
2244 
2245 	ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error);
2246 	/* SHUTDOWN may require DF_FLUSH */
2247 	if (*error == SEV_RET_DFFLUSH_REQUIRED) {
2248 		int dfflush_error = SEV_RET_NO_FW_CALL;
2249 
2250 		ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error);
2251 		if (ret) {
2252 			dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n",
2253 				ret, dfflush_error);
2254 			return ret;
2255 		}
2256 		/* reissue the shutdown command */
2257 		ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data,
2258 					  error);
2259 	}
2260 	if (ret) {
2261 		dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n",
2262 			ret, *error);
2263 		return ret;
2264 	}
2265 
2266 	if (data.x86_snp_shutdown) {
2267 		if (!panic)
2268 			snp_shutdown();
2269 		snp_hv_fixed_pages_state_update(sev, ALLOCATED);
2270 	} else {
2271 		/*
2272 		 * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP
2273 		 * enforcement by the IOMMU and also transitions all pages
2274 		 * associated with the IOMMU to the Reclaim state.
2275 		 * Firmware was transitioning the IOMMU pages to Hypervisor state
2276 		 * before version 1.53. But, accounting for the number of assigned
2277 		 * 4kB pages in a 2M page was done incorrectly by not transitioning
2278 		 * to the Reclaim state. This resulted in RMP #PF when later accessing
2279 		 * the 2M page containing those pages during kexec boot. Hence, the
2280 		 * firmware now transitions these pages to Reclaim state and hypervisor
2281 		 * needs to transition these pages to shared state. SNP Firmware
2282 		 * version 1.53 and above are needed for kexec boot.
2283 		 */
2284 		ret = amd_iommu_snp_disable();
2285 		if (ret) {
2286 			dev_err(sev->dev, "SNP IOMMU shutdown failed\n");
2287 			return ret;
2288 		}
2289 	}
2290 
2291 	snp_leak_hv_fixed_pages();
2292 	sev->snp_initialized = false;
2293 	dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n");
2294 
2295 	/*
2296 	 * __sev_snp_shutdown_locked() deadlocks when it tries to unregister
2297 	 * itself during panic as the panic notifier is called with RCU read
2298 	 * lock held and notifier unregistration does RCU synchronization.
2299 	 */
2300 	if (!panic)
2301 		atomic_notifier_chain_unregister(&panic_notifier_list,
2302 						 &snp_panic_notifier);
2303 
2304 	/* Reset TMR size back to default */
2305 	sev_es_tmr_size = SEV_TMR_SIZE;
2306 
2307 	return ret;
2308 }
2309 
2310 static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable)
2311 {
2312 	struct sev_device *sev = psp_master->sev_data;
2313 	struct sev_user_data_pek_cert_import input;
2314 	struct sev_data_pek_cert_import data;
2315 	bool shutdown_required = false;
2316 	void *pek_blob, *oca_blob;
2317 	int ret;
2318 
2319 	if (!writable)
2320 		return -EPERM;
2321 
2322 	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2323 		return -EFAULT;
2324 
2325 	/* copy PEK certificate blobs from userspace */
2326 	pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len);
2327 	if (IS_ERR(pek_blob))
2328 		return PTR_ERR(pek_blob);
2329 
2330 	data.reserved = 0;
2331 	data.pek_cert_address = __psp_pa(pek_blob);
2332 	data.pek_cert_len = input.pek_cert_len;
2333 
2334 	/* copy PEK certificate blobs from userspace */
2335 	oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len);
2336 	if (IS_ERR(oca_blob)) {
2337 		ret = PTR_ERR(oca_blob);
2338 		goto e_free_pek;
2339 	}
2340 
2341 	data.oca_cert_address = __psp_pa(oca_blob);
2342 	data.oca_cert_len = input.oca_cert_len;
2343 
2344 	mutex_lock(&sev_cmd_mutex);
2345 
2346 	/* If platform is not in INIT state then transition it to INIT */
2347 	if (sev->sev_plat_status.state != SEV_STATE_INIT) {
2348 		ret = sev_move_to_init_state(argp, &shutdown_required);
2349 		if (ret) {
2350 			mutex_unlock(&sev_cmd_mutex);
2351 			/* No shutdown is needed if init fails */
2352 			goto e_free_oca;
2353 		}
2354 	}
2355 
2356 	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error);
2357 
2358 	if (shutdown_required)
2359 		__sev_firmware_shutdown(sev, false);
2360 
2361 	mutex_unlock(&sev_cmd_mutex);
2362 
2363 e_free_oca:
2364 	kfree(oca_blob);
2365 e_free_pek:
2366 	kfree(pek_blob);
2367 	return ret;
2368 }
2369 
2370 static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp)
2371 {
2372 	struct sev_user_data_get_id2 input;
2373 	struct sev_data_get_id data;
2374 	void __user *input_address;
2375 	void *id_blob = NULL;
2376 	int ret;
2377 
2378 	/* SEV GET_ID is available from SEV API v0.16 and up */
2379 	if (!sev_version_greater_or_equal(0, 16))
2380 		return -ENOTSUPP;
2381 
2382 	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2383 		return -EFAULT;
2384 
2385 	input_address = (void __user *)input.address;
2386 
2387 	if (input.address && input.length) {
2388 		/*
2389 		 * The length of the ID shouldn't be assumed by software since
2390 		 * it may change in the future.  The allocation size is limited
2391 		 * to 1 << (PAGE_SHIFT + MAX_PAGE_ORDER) by the page allocator.
2392 		 * If the allocation fails, simply return ENOMEM rather than
2393 		 * warning in the kernel log.
2394 		 */
2395 		id_blob = kzalloc(input.length, GFP_KERNEL | __GFP_NOWARN);
2396 		if (!id_blob)
2397 			return -ENOMEM;
2398 
2399 		data.address = __psp_pa(id_blob);
2400 		data.len = input.length;
2401 	} else {
2402 		data.address = 0;
2403 		data.len = 0;
2404 	}
2405 
2406 	mutex_lock(&sev_cmd_mutex);
2407 	ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, &data, &argp->error);
2408 	mutex_unlock(&sev_cmd_mutex);
2409 
2410 	/*
2411 	 * Firmware will return the length of the ID value (either the minimum
2412 	 * required length or the actual length written), return it to the user.
2413 	 */
2414 	input.length = data.len;
2415 
2416 	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2417 		ret = -EFAULT;
2418 		goto e_free;
2419 	}
2420 
2421 	if (ret || WARN_ON_ONCE(argp->error))
2422 		goto e_free;
2423 
2424 	if (id_blob) {
2425 		if (copy_to_user(input_address, id_blob, data.len)) {
2426 			ret = -EFAULT;
2427 			goto e_free;
2428 		}
2429 	}
2430 
2431 e_free:
2432 	kfree(id_blob);
2433 
2434 	return ret;
2435 }
2436 
2437 static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp)
2438 {
2439 	struct sev_data_get_id *data;
2440 	u64 data_size, user_size;
2441 	void *id_blob, *mem;
2442 	int ret;
2443 
2444 	/* SEV GET_ID available from SEV API v0.16 and up */
2445 	if (!sev_version_greater_or_equal(0, 16))
2446 		return -ENOTSUPP;
2447 
2448 	/* SEV FW expects the buffer it fills with the ID to be
2449 	 * 8-byte aligned. Memory allocated should be enough to
2450 	 * hold data structure + alignment padding + memory
2451 	 * where SEV FW writes the ID.
2452 	 */
2453 	data_size = ALIGN(sizeof(struct sev_data_get_id), 8);
2454 	user_size = sizeof(struct sev_user_data_get_id);
2455 
2456 	mem = kzalloc(data_size + user_size, GFP_KERNEL);
2457 	if (!mem)
2458 		return -ENOMEM;
2459 
2460 	data = mem;
2461 	id_blob = mem + data_size;
2462 
2463 	data->address = __psp_pa(id_blob);
2464 	data->len = user_size;
2465 
2466 	mutex_lock(&sev_cmd_mutex);
2467 	ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error);
2468 	mutex_unlock(&sev_cmd_mutex);
2469 	if (!ret) {
2470 		if (copy_to_user((void __user *)argp->data, id_blob, data->len))
2471 			ret = -EFAULT;
2472 	}
2473 
2474 	kfree(mem);
2475 
2476 	return ret;
2477 }
2478 
2479 static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
2480 {
2481 	struct sev_device *sev = psp_master->sev_data;
2482 	struct sev_user_data_pdh_cert_export input;
2483 	void *pdh_blob = NULL, *cert_blob = NULL;
2484 	struct sev_data_pdh_cert_export data;
2485 	void __user *input_cert_chain_address;
2486 	void __user *input_pdh_cert_address;
2487 	bool shutdown_required = false;
2488 	int ret;
2489 
2490 	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2491 		return -EFAULT;
2492 
2493 	memset(&data, 0, sizeof(data));
2494 
2495 	input_pdh_cert_address = (void __user *)input.pdh_cert_address;
2496 	input_cert_chain_address = (void __user *)input.cert_chain_address;
2497 
2498 	/* Userspace wants to query the certificate length. */
2499 	if (!input.pdh_cert_address ||
2500 	    !input.pdh_cert_len ||
2501 	    !input.cert_chain_address ||
2502 	    !input.cert_chain_len)
2503 		goto cmd;
2504 
2505 	/* Allocate a physically contiguous buffer to store the PDH blob. */
2506 	if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE)
2507 		return -EFAULT;
2508 
2509 	/* Allocate a physically contiguous buffer to store the cert chain blob. */
2510 	if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE)
2511 		return -EFAULT;
2512 
2513 	pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL);
2514 	if (!pdh_blob)
2515 		return -ENOMEM;
2516 
2517 	data.pdh_cert_address = __psp_pa(pdh_blob);
2518 	data.pdh_cert_len = input.pdh_cert_len;
2519 
2520 	cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL);
2521 	if (!cert_blob) {
2522 		ret = -ENOMEM;
2523 		goto e_free_pdh;
2524 	}
2525 
2526 	data.cert_chain_address = __psp_pa(cert_blob);
2527 	data.cert_chain_len = input.cert_chain_len;
2528 
2529 cmd:
2530 	mutex_lock(&sev_cmd_mutex);
2531 
2532 	/* If platform is not in INIT state then transition it to INIT. */
2533 	if (sev->sev_plat_status.state != SEV_STATE_INIT) {
2534 		if (!writable) {
2535 			ret = -EPERM;
2536 			mutex_unlock(&sev_cmd_mutex);
2537 			goto e_free_cert;
2538 		}
2539 		ret = sev_move_to_init_state(argp, &shutdown_required);
2540 		if (ret) {
2541 			/* No shutdown is needed if init fails */
2542 			mutex_unlock(&sev_cmd_mutex);
2543 			goto e_free_cert;
2544 		}
2545 	}
2546 
2547 	ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error);
2548 
2549 	if (shutdown_required)
2550 		__sev_firmware_shutdown(sev, false);
2551 
2552 	mutex_unlock(&sev_cmd_mutex);
2553 
2554 	/*
2555 	 * Firmware will return the length of the blobs (either the minimum
2556 	 * required length or the actual length written), return 'em to the user.
2557 	 */
2558 	input.cert_chain_len = data.cert_chain_len;
2559 	input.pdh_cert_len = data.pdh_cert_len;
2560 
2561 	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2562 		ret = -EFAULT;
2563 		goto e_free_cert;
2564 	}
2565 
2566 	if (ret || WARN_ON_ONCE(argp->error))
2567 		goto e_free_cert;
2568 
2569 	if (pdh_blob) {
2570 		if (copy_to_user(input_pdh_cert_address,
2571 				 pdh_blob, input.pdh_cert_len)) {
2572 			ret = -EFAULT;
2573 			goto e_free_cert;
2574 		}
2575 	}
2576 
2577 	if (cert_blob) {
2578 		if (copy_to_user(input_cert_chain_address,
2579 				 cert_blob, input.cert_chain_len))
2580 			ret = -EFAULT;
2581 	}
2582 
2583 e_free_cert:
2584 	kfree(cert_blob);
2585 e_free_pdh:
2586 	kfree(pdh_blob);
2587 	return ret;
2588 }
2589 
2590 static int __sev_do_snp_platform_status(struct sev_user_data_snp_status *status,
2591 					int *error)
2592 {
2593 	struct sev_device *sev = psp_master->sev_data;
2594 	struct sev_data_snp_addr buf;
2595 	struct page *status_page;
2596 	void *data;
2597 	int ret;
2598 
2599 	status_page = alloc_page(GFP_KERNEL_ACCOUNT);
2600 	if (!status_page)
2601 		return -ENOMEM;
2602 
2603 	data = page_address(status_page);
2604 
2605 	mutex_lock(&sev_cmd_mutex);
2606 
2607 	/*
2608 	 * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed
2609 	 * when SNP has been initialized, the status page must be firmware-owned.
2610 	 */
2611 	if (sev->snp_initialized) {
2612 		/*
2613 		 * Firmware expects the status page to be in Firmware state,
2614 		 * otherwise it will report an error INVALID_PAGE_STATE.
2615 		 */
2616 		if (rmp_mark_pages_firmware(__pa(data), 1, true)) {
2617 			ret = -EFAULT;
2618 			mutex_unlock(&sev_cmd_mutex);
2619 			goto cleanup;
2620 		}
2621 	}
2622 
2623 	buf.address = __psp_pa(data);
2624 	ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error);
2625 
2626 	if (sev->snp_initialized) {
2627 		/*
2628 		 * The status page will be in Reclaim state on success, or left
2629 		 * in Firmware state on failure. Use snp_reclaim_pages() to
2630 		 * transition either case back to Hypervisor-owned state.
2631 		 */
2632 		if (snp_reclaim_pages(__pa(data), 1, true)) {
2633 			mutex_unlock(&sev_cmd_mutex);
2634 			return -EFAULT;
2635 		}
2636 	}
2637 
2638 	mutex_unlock(&sev_cmd_mutex);
2639 
2640 	if (ret)
2641 		goto cleanup;
2642 
2643 	memcpy(status, data, sizeof(*status));
2644 
2645 cleanup:
2646 	__free_pages(status_page, 0);
2647 	return ret;
2648 }
2649 
2650 static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
2651 {
2652 	struct sev_user_data_snp_status status;
2653 	int ret;
2654 
2655 	if (!argp->data)
2656 		return -EINVAL;
2657 
2658 	ret = __sev_do_snp_platform_status(&status, &argp->error);
2659 	if (ret < 0)
2660 		return ret;
2661 
2662 	if (copy_to_user((void __user *)argp->data, &status,
2663 			 sizeof(struct sev_user_data_snp_status)))
2664 		ret = -EFAULT;
2665 
2666 	return ret;
2667 }
2668 
2669 static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp)
2670 {
2671 	struct sev_data_snp_commit buf;
2672 	int ret;
2673 
2674 	buf.len = sizeof(buf);
2675 
2676 	mutex_lock(&sev_cmd_mutex);
2677 	ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error);
2678 	mutex_unlock(&sev_cmd_mutex);
2679 
2680 	return ret;
2681 }
2682 
2683 static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable)
2684 {
2685 	struct sev_device *sev = psp_master->sev_data;
2686 	struct sev_user_data_snp_config config;
2687 	int ret;
2688 
2689 	if (!argp->data)
2690 		return -EINVAL;
2691 
2692 	if (!writable)
2693 		return -EPERM;
2694 
2695 	if (!sev->snp_initialized)
2696 		return -ENODEV;
2697 
2698 	if (copy_from_user(&config, (void __user *)argp->data, sizeof(config)))
2699 		return -EFAULT;
2700 
2701 	mutex_lock(&sev_cmd_mutex);
2702 	ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error);
2703 	mutex_unlock(&sev_cmd_mutex);
2704 
2705 	return ret;
2706 }
2707 
2708 static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
2709 {
2710 	struct sev_device *sev = psp_master->sev_data;
2711 	struct sev_user_data_snp_vlek_load input;
2712 	void *blob;
2713 	int ret;
2714 
2715 	if (!argp->data)
2716 		return -EINVAL;
2717 
2718 	if (!writable)
2719 		return -EPERM;
2720 
2721 	if (!sev->snp_initialized)
2722 		return -ENODEV;
2723 
2724 	if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input)))
2725 		return -EFAULT;
2726 
2727 	if (input.len != sizeof(input) || input.vlek_wrapped_version != 0)
2728 		return -EINVAL;
2729 
2730 	blob = psp_copy_user_blob(input.vlek_wrapped_address,
2731 				  sizeof(struct sev_user_data_snp_wrapped_vlek_hashstick));
2732 	if (IS_ERR(blob))
2733 		return PTR_ERR(blob);
2734 
2735 	input.vlek_wrapped_address = __psp_pa(blob);
2736 
2737 	mutex_lock(&sev_cmd_mutex);
2738 	ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error);
2739 	mutex_unlock(&sev_cmd_mutex);
2740 
2741 	kfree(blob);
2742 
2743 	return ret;
2744 }
2745 
2746 static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
2747 {
2748 	void __user *argp = (void __user *)arg;
2749 	struct sev_issue_cmd input;
2750 	int ret = -EFAULT;
2751 	bool writable = file->f_mode & FMODE_WRITE;
2752 
2753 	if (!psp_master || !psp_master->sev_data)
2754 		return -ENODEV;
2755 
2756 	if (ioctl != SEV_ISSUE_CMD)
2757 		return -EINVAL;
2758 
2759 	if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd)))
2760 		return -EFAULT;
2761 
2762 	if (input.cmd > SEV_MAX)
2763 		return -EINVAL;
2764 
2765 	switch (input.cmd) {
2766 
2767 	case SEV_FACTORY_RESET:
2768 		ret = sev_ioctl_do_reset(&input, writable);
2769 		break;
2770 	case SEV_PLATFORM_STATUS:
2771 		ret = sev_ioctl_do_platform_status(&input);
2772 		break;
2773 	case SEV_PEK_GEN:
2774 		ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input, writable);
2775 		break;
2776 	case SEV_PDH_GEN:
2777 		ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input, writable);
2778 		break;
2779 	case SEV_PEK_CSR:
2780 		ret = sev_ioctl_do_pek_csr(&input, writable);
2781 		break;
2782 	case SEV_PEK_CERT_IMPORT:
2783 		ret = sev_ioctl_do_pek_import(&input, writable);
2784 		break;
2785 	case SEV_PDH_CERT_EXPORT:
2786 		ret = sev_ioctl_do_pdh_export(&input, writable);
2787 		break;
2788 	case SEV_GET_ID:
2789 		pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n");
2790 		ret = sev_ioctl_do_get_id(&input);
2791 		break;
2792 	case SEV_GET_ID2:
2793 		ret = sev_ioctl_do_get_id2(&input);
2794 		break;
2795 	case SNP_PLATFORM_STATUS:
2796 		ret = sev_ioctl_do_snp_platform_status(&input);
2797 		break;
2798 	case SNP_COMMIT:
2799 		ret = sev_ioctl_do_snp_commit(&input);
2800 		break;
2801 	case SNP_SET_CONFIG:
2802 		ret = sev_ioctl_do_snp_set_config(&input, writable);
2803 		break;
2804 	case SNP_VLEK_LOAD:
2805 		ret = sev_ioctl_do_snp_vlek_load(&input, writable);
2806 		break;
2807 	default:
2808 		ret = -EINVAL;
2809 		goto out;
2810 	}
2811 
2812 	if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd)))
2813 		ret = -EFAULT;
2814 out:
2815 
2816 	return ret;
2817 }
2818 
2819 static const struct file_operations sev_fops = {
2820 	.owner	= THIS_MODULE,
2821 	.unlocked_ioctl = sev_ioctl,
2822 };
2823 
2824 int sev_platform_status(struct sev_user_data_status *data, int *error)
2825 {
2826 	return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error);
2827 }
2828 EXPORT_SYMBOL_GPL(sev_platform_status);
2829 
2830 int sev_guest_deactivate(struct sev_data_deactivate *data, int *error)
2831 {
2832 	return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error);
2833 }
2834 EXPORT_SYMBOL_GPL(sev_guest_deactivate);
2835 
2836 int sev_guest_activate(struct sev_data_activate *data, int *error)
2837 {
2838 	return sev_do_cmd(SEV_CMD_ACTIVATE, data, error);
2839 }
2840 EXPORT_SYMBOL_GPL(sev_guest_activate);
2841 
2842 int sev_guest_decommission(struct sev_data_decommission *data, int *error)
2843 {
2844 	return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error);
2845 }
2846 EXPORT_SYMBOL_GPL(sev_guest_decommission);
2847 
2848 int sev_guest_df_flush(int *error)
2849 {
2850 	return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error);
2851 }
2852 EXPORT_SYMBOL_GPL(sev_guest_df_flush);
2853 
2854 static void sev_exit(struct kref *ref)
2855 {
2856 	misc_deregister(&misc_dev->misc);
2857 	kfree(misc_dev);
2858 	misc_dev = NULL;
2859 }
2860 
2861 static int sev_misc_init(struct sev_device *sev)
2862 {
2863 	struct device *dev = sev->dev;
2864 	int ret;
2865 
2866 	/*
2867 	 * SEV feature support can be detected on multiple devices but the SEV
2868 	 * FW commands must be issued on the master. During probe, we do not
2869 	 * know the master hence we create /dev/sev on the first device probe.
2870 	 * sev_do_cmd() finds the right master device to which to issue the
2871 	 * command to the firmware.
2872 	 */
2873 	if (!misc_dev) {
2874 		struct miscdevice *misc;
2875 
2876 		misc_dev = kzalloc_obj(*misc_dev);
2877 		if (!misc_dev)
2878 			return -ENOMEM;
2879 
2880 		misc = &misc_dev->misc;
2881 		misc->minor = MISC_DYNAMIC_MINOR;
2882 		misc->name = DEVICE_NAME;
2883 		misc->fops = &sev_fops;
2884 
2885 		ret = misc_register(misc);
2886 		if (ret)
2887 			return ret;
2888 
2889 		kref_init(&misc_dev->refcount);
2890 	} else {
2891 		kref_get(&misc_dev->refcount);
2892 	}
2893 
2894 	init_waitqueue_head(&sev->int_queue);
2895 	sev->misc = misc_dev;
2896 	dev_dbg(dev, "registered SEV device\n");
2897 
2898 	return 0;
2899 }
2900 
2901 int sev_dev_init(struct psp_device *psp)
2902 {
2903 	struct device *dev = psp->dev;
2904 	struct sev_device *sev;
2905 	int ret = -ENOMEM;
2906 
2907 	if (!boot_cpu_has(X86_FEATURE_SEV)) {
2908 		dev_info_once(dev, "SEV: memory encryption not enabled by BIOS\n");
2909 		return 0;
2910 	}
2911 
2912 	sev = devm_kzalloc_obj(dev, *sev);
2913 	if (!sev)
2914 		goto e_err;
2915 
2916 	sev->cmd_buf = (void *)devm_get_free_pages(dev, GFP_KERNEL, 1);
2917 	if (!sev->cmd_buf)
2918 		goto e_sev;
2919 
2920 	sev->cmd_buf_backup = (uint8_t *)sev->cmd_buf + PAGE_SIZE;
2921 
2922 	psp->sev_data = sev;
2923 
2924 	sev->dev = dev;
2925 	sev->psp = psp;
2926 
2927 	sev->io_regs = psp->io_regs;
2928 
2929 	sev->vdata = (struct sev_vdata *)psp->vdata->sev;
2930 	if (!sev->vdata) {
2931 		ret = -ENODEV;
2932 		dev_err(dev, "sev: missing driver data\n");
2933 		goto e_buf;
2934 	}
2935 
2936 	psp_set_sev_irq_handler(psp, sev_irq_handler, sev);
2937 
2938 	ret = sev_misc_init(sev);
2939 	if (ret)
2940 		goto e_irq;
2941 
2942 	dev_notice(dev, "sev enabled\n");
2943 
2944 	return 0;
2945 
2946 e_irq:
2947 	psp_clear_sev_irq_handler(psp);
2948 e_buf:
2949 	devm_free_pages(dev, (unsigned long)sev->cmd_buf);
2950 e_sev:
2951 	devm_kfree(dev, sev);
2952 e_err:
2953 	psp->sev_data = NULL;
2954 
2955 	dev_notice(dev, "sev initialization failed\n");
2956 
2957 	return ret;
2958 }
2959 
2960 static void __sev_firmware_shutdown(struct sev_device *sev, bool panic)
2961 {
2962 	int error;
2963 
2964 	__sev_platform_shutdown_locked(&error);
2965 
2966 	if (sev_es_tmr) {
2967 		/*
2968 		 * The TMR area was encrypted, flush it from the cache.
2969 		 *
2970 		 * If invoked during panic handling, local interrupts are
2971 		 * disabled and all CPUs are stopped, so wbinvd_on_all_cpus()
2972 		 * can't be used. In that case, wbinvd() is done on remote CPUs
2973 		 * via the NMI callback, and done for this CPU later during
2974 		 * SNP shutdown, so wbinvd_on_all_cpus() can be skipped.
2975 		 */
2976 		if (!panic)
2977 			wbinvd_on_all_cpus();
2978 
2979 		__snp_free_firmware_pages(virt_to_page(sev_es_tmr),
2980 					  get_order(sev_es_tmr_size),
2981 					  true);
2982 		sev_es_tmr = NULL;
2983 	}
2984 
2985 	if (sev_init_ex_buffer) {
2986 		__snp_free_firmware_pages(virt_to_page(sev_init_ex_buffer),
2987 					  get_order(NV_LENGTH),
2988 					  true);
2989 		sev_init_ex_buffer = NULL;
2990 	}
2991 
2992 	__sev_snp_shutdown_locked(&error, panic);
2993 }
2994 
2995 static void sev_firmware_shutdown(struct sev_device *sev)
2996 {
2997 	/*
2998 	 * Calling without sev_cmd_mutex held as TSM will likely try disconnecting
2999 	 * IDE and this ends up calling sev_do_cmd() which locks sev_cmd_mutex.
3000 	 */
3001 	if (sev->tio_status)
3002 		sev_tsm_uninit(sev);
3003 
3004 	/*
3005 	 * Remove the sysfs interface before taking the sev_cmd_mutex.
3006 	 * sysfs_remove_group() waits for in-flight _show()/_store() handlers
3007 	 * to drain, and those handlers issue SNP_VERIFY_MITIGATION via
3008 	 * sev_do_cmd() which acquires the sev_cmd_mutex. Removing the group
3009 	 * while holding the mutex could therefore deadlock.
3010 	 */
3011 	sev_snp_unregister_verify_mitigation(sev);
3012 
3013 	mutex_lock(&sev_cmd_mutex);
3014 
3015 	__sev_firmware_shutdown(sev, false);
3016 
3017 	kfree(sev->tio_status);
3018 	sev->tio_status = NULL;
3019 
3020 	mutex_unlock(&sev_cmd_mutex);
3021 }
3022 
3023 void sev_platform_shutdown(void)
3024 {
3025 	if (!psp_master || !psp_master->sev_data)
3026 		return;
3027 
3028 	sev_firmware_shutdown(psp_master->sev_data);
3029 }
3030 EXPORT_SYMBOL_GPL(sev_platform_shutdown);
3031 
3032 u64 sev_get_snp_policy_bits(void)
3033 {
3034 	struct psp_device *psp = psp_master;
3035 	struct sev_device *sev;
3036 	u64 policy_bits;
3037 
3038 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
3039 		return 0;
3040 
3041 	if (!psp || !psp->sev_data)
3042 		return 0;
3043 
3044 	sev = psp->sev_data;
3045 
3046 	policy_bits = SNP_POLICY_MASK_BASE;
3047 
3048 	if (sev->snp_plat_status.feature_info) {
3049 		if (sev->snp_feat_info_0.ecx & SNP_RAPL_DISABLE_SUPPORTED)
3050 			policy_bits |= SNP_POLICY_MASK_RAPL_DIS;
3051 
3052 		if (sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED)
3053 			policy_bits |= SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM;
3054 
3055 		if (sev->snp_feat_info_0.ecx & SNP_AES_256_XTS_POLICY_SUPPORTED)
3056 			policy_bits |= SNP_POLICY_MASK_MEM_AES_256_XTS;
3057 
3058 		if (sev->snp_feat_info_0.ecx & SNP_CXL_ALLOW_POLICY_SUPPORTED)
3059 			policy_bits |= SNP_POLICY_MASK_CXL_ALLOW;
3060 
3061 		if (sev_version_greater_or_equal(1, 58))
3062 			policy_bits |= SNP_POLICY_MASK_PAGE_SWAP_DISABLE;
3063 	}
3064 
3065 	return policy_bits;
3066 }
3067 EXPORT_SYMBOL_GPL(sev_get_snp_policy_bits);
3068 
3069 void sev_dev_destroy(struct psp_device *psp)
3070 {
3071 	struct sev_device *sev = psp->sev_data;
3072 
3073 	if (!sev)
3074 		return;
3075 
3076 	sev_firmware_shutdown(sev);
3077 
3078 	if (sev->misc)
3079 		kref_put(&misc_dev->refcount, sev_exit);
3080 
3081 	psp_clear_sev_irq_handler(psp);
3082 }
3083 
3084 static int snp_shutdown_on_panic(struct notifier_block *nb,
3085 				 unsigned long reason, void *arg)
3086 {
3087 	struct sev_device *sev = psp_master->sev_data;
3088 
3089 	/*
3090 	 * If sev_cmd_mutex is already acquired, then it's likely
3091 	 * another PSP command is in flight and issuing a shutdown
3092 	 * would fail in unexpected ways. Rather than create even
3093 	 * more confusion during a panic, just bail out here.
3094 	 */
3095 	if (mutex_is_locked(&sev_cmd_mutex))
3096 		return NOTIFY_DONE;
3097 
3098 	__sev_firmware_shutdown(sev, true);
3099 
3100 	return NOTIFY_DONE;
3101 }
3102 
3103 int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd,
3104 				void *data, int *error)
3105 {
3106 	if (!filep || filep->f_op != &sev_fops)
3107 		return -EBADF;
3108 
3109 	return sev_do_cmd(cmd, data, error);
3110 }
3111 EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
3112 
3113 void sev_pci_init(void)
3114 {
3115 	struct sev_device *sev = psp_master->sev_data;
3116 	u8 api_major, api_minor, build;
3117 
3118 	if (!sev)
3119 		return;
3120 
3121 	psp_timeout = psp_probe_timeout;
3122 
3123 	if (sev_get_api_version())
3124 		goto err;
3125 
3126 	api_major = sev->api_major;
3127 	api_minor = sev->api_minor;
3128 	build     = sev->build;
3129 
3130 	if (sev_update_firmware(sev->dev) == 0)
3131 		sev_get_api_version();
3132 
3133 	if (api_major != sev->api_major || api_minor != sev->api_minor ||
3134 	    build != sev->build)
3135 		dev_info(sev->dev, "SEV firmware updated from %d.%d.%d to %d.%d.%d\n",
3136 			 api_major, api_minor, build,
3137 			 sev->api_major, sev->api_minor, sev->build);
3138 
3139 	return;
3140 
3141 err:
3142 	sev_dev_destroy(psp_master);
3143 
3144 	psp_master->sev_data = NULL;
3145 }
3146 
3147 void sev_pci_exit(void)
3148 {
3149 	struct sev_device *sev = psp_master->sev_data;
3150 
3151 	if (!sev)
3152 		return;
3153 
3154 	sev_firmware_shutdown(sev);
3155 }
3156 
3157 static int get_v1_svn(struct sev_device *sev)
3158 {
3159 	struct sev_snp_tcb_version_genoa_milan *tcb;
3160 	struct sev_user_data_snp_status status;
3161 	int ret, error = 0;
3162 
3163 	ret = __sev_do_snp_platform_status(&status, &error);
3164 	if (ret < 0)
3165 		return ret;
3166 
3167 	tcb = (struct sev_snp_tcb_version_genoa_milan *)&status
3168 		      .current_tcb_version;
3169 	return tcb->snp;
3170 }
3171 
3172 static int get_v2_svn(struct sev_device *sev)
3173 {
3174 	struct sev_user_data_snp_status status;
3175 	struct sev_snp_tcb_version_turin *tcb;
3176 	int ret, error = 0;
3177 
3178 	ret = __sev_do_snp_platform_status(&status, &error);
3179 	if (ret < 0)
3180 		return ret;
3181 
3182 	tcb = (struct sev_snp_tcb_version_turin *)&status
3183 		      .current_tcb_version;
3184 	return tcb->snp;
3185 }
3186 
3187 static bool sev_firmware_allows_es(struct sev_device *sev)
3188 {
3189 	/* Documented in AMD-SB-3023 */
3190 	if (boot_cpu_has(X86_FEATURE_ZEN4) || boot_cpu_has(X86_FEATURE_ZEN3))
3191 		return get_v1_svn(sev) < 0x1b;
3192 	else if (boot_cpu_has(X86_FEATURE_ZEN5))
3193 		return get_v2_svn(sev) < 0x4;
3194 	else
3195 		return true;
3196 }
3197 
3198 int sev_firmware_supported_vm_types(void)
3199 {
3200 	int supported_vm_types = 0;
3201 	struct sev_device *sev;
3202 
3203 	if (!psp_master || !psp_master->sev_data)
3204 		return supported_vm_types;
3205 	sev = psp_master->sev_data;
3206 
3207 	supported_vm_types |= BIT(KVM_X86_SEV_VM);
3208 	supported_vm_types |= BIT(KVM_X86_SEV_ES_VM);
3209 
3210 	if (!sev->snp_initialized)
3211 		return supported_vm_types;
3212 
3213 	supported_vm_types |= BIT(KVM_X86_SNP_VM);
3214 
3215 	if (!sev_firmware_allows_es(sev))
3216 		supported_vm_types &= ~BIT(KVM_X86_SEV_ES_VM);
3217 
3218 	return supported_vm_types;
3219 
3220 }
3221 EXPORT_SYMBOL_FOR_MODULES(sev_firmware_supported_vm_types, "kvm-amd");
3222