xref: /linux/arch/x86/platform/efi/efi.c (revision 680e6ffa15103ab610c0fc1241d2f98c801b13e2)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Common EFI (Extensible Firmware Interface) support functions
4  * Based on Extensible Firmware Interface Specification version 1.0
5  *
6  * Copyright (C) 1999 VA Linux Systems
7  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
8  * Copyright (C) 1999-2002 Hewlett-Packard Co.
9  *	David Mosberger-Tang <davidm@hpl.hp.com>
10  *	Stephane Eranian <eranian@hpl.hp.com>
11  * Copyright (C) 2005-2008 Intel Co.
12  *	Fenghua Yu <fenghua.yu@intel.com>
13  *	Bibo Mao <bibo.mao@intel.com>
14  *	Chandramouli Narayanan <mouli@linux.intel.com>
15  *	Huang Ying <ying.huang@intel.com>
16  * Copyright (C) 2013 SuSE Labs
17  *	Borislav Petkov <bp@suse.de> - runtime services VA mapping
18  *
19  * Copied from efi_32.c to eliminate the duplicated code between EFI
20  * 32/64 support code. --ying 2007-10-26
21  *
22  * All EFI Runtime Services are not implemented yet as EFI only
23  * supports physical mode addressing on SoftSDV. This is to be fixed
24  * in a future version.  --drummond 1999-07-20
25  *
26  * Implemented EFI runtime services and virtual mode calls.  --davidm
27  *
28  * Goutham Rao: <goutham.rao@intel.com>
29  *	Skip non-WB memory and ignore empty memory ranges.
30  */
31 
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 
34 #include <linux/kernel.h>
35 #include <linux/init.h>
36 #include <linux/efi.h>
37 #include <linux/efi-bgrt.h>
38 #include <linux/export.h>
39 #include <linux/memblock.h>
40 #include <linux/slab.h>
41 #include <linux/spinlock.h>
42 #include <linux/uaccess.h>
43 #include <linux/time.h>
44 #include <linux/io.h>
45 #include <linux/reboot.h>
46 #include <linux/bcd.h>
47 
48 #include <asm/setup.h>
49 #include <asm/efi.h>
50 #include <asm/e820/api.h>
51 #include <asm/time.h>
52 #include <asm/set_memory.h>
53 #include <asm/tlbflush.h>
54 #include <asm/x86_init.h>
55 #include <asm/uv/uv.h>
56 
57 static struct efi efi_phys __initdata;
58 static efi_system_table_t efi_systab __initdata;
59 
60 static efi_config_table_type_t arch_tables[] __initdata = {
61 #ifdef CONFIG_X86_UV
62 	{UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
63 #endif
64 	{NULL_GUID, NULL, NULL},
65 };
66 
67 u64 efi_setup;		/* efi setup_data physical address */
68 
69 static int add_efi_memmap __initdata;
70 static int __init setup_add_efi_memmap(char *arg)
71 {
72 	add_efi_memmap = 1;
73 	return 0;
74 }
75 early_param("add_efi_memmap", setup_add_efi_memmap);
76 
77 static efi_status_t __init phys_efi_set_virtual_address_map(
78 	unsigned long memory_map_size,
79 	unsigned long descriptor_size,
80 	u32 descriptor_version,
81 	efi_memory_desc_t *virtual_map)
82 {
83 	efi_status_t status;
84 	unsigned long flags;
85 	pgd_t *save_pgd;
86 
87 	save_pgd = efi_call_phys_prolog();
88 
89 	/* Disable interrupts around EFI calls: */
90 	local_irq_save(flags);
91 	status = efi_call_phys(efi_phys.set_virtual_address_map,
92 			       memory_map_size, descriptor_size,
93 			       descriptor_version, virtual_map);
94 	local_irq_restore(flags);
95 
96 	efi_call_phys_epilog(save_pgd);
97 
98 	return status;
99 }
100 
101 void __init efi_find_mirror(void)
102 {
103 	efi_memory_desc_t *md;
104 	u64 mirror_size = 0, total_size = 0;
105 
106 	for_each_efi_memory_desc(md) {
107 		unsigned long long start = md->phys_addr;
108 		unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
109 
110 		total_size += size;
111 		if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
112 			memblock_mark_mirror(start, size);
113 			mirror_size += size;
114 		}
115 	}
116 	if (mirror_size)
117 		pr_info("Memory: %lldM/%lldM mirrored memory\n",
118 			mirror_size>>20, total_size>>20);
119 }
120 
121 /*
122  * Tell the kernel about the EFI memory map.  This might include
123  * more than the max 128 entries that can fit in the e820 legacy
124  * (zeropage) memory map.
125  */
126 
127 static void __init do_add_efi_memmap(void)
128 {
129 	efi_memory_desc_t *md;
130 
131 	for_each_efi_memory_desc(md) {
132 		unsigned long long start = md->phys_addr;
133 		unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
134 		int e820_type;
135 
136 		switch (md->type) {
137 		case EFI_LOADER_CODE:
138 		case EFI_LOADER_DATA:
139 		case EFI_BOOT_SERVICES_CODE:
140 		case EFI_BOOT_SERVICES_DATA:
141 		case EFI_CONVENTIONAL_MEMORY:
142 			if (md->attribute & EFI_MEMORY_WB)
143 				e820_type = E820_TYPE_RAM;
144 			else
145 				e820_type = E820_TYPE_RESERVED;
146 			break;
147 		case EFI_ACPI_RECLAIM_MEMORY:
148 			e820_type = E820_TYPE_ACPI;
149 			break;
150 		case EFI_ACPI_MEMORY_NVS:
151 			e820_type = E820_TYPE_NVS;
152 			break;
153 		case EFI_UNUSABLE_MEMORY:
154 			e820_type = E820_TYPE_UNUSABLE;
155 			break;
156 		case EFI_PERSISTENT_MEMORY:
157 			e820_type = E820_TYPE_PMEM;
158 			break;
159 		default:
160 			/*
161 			 * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
162 			 * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
163 			 * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
164 			 */
165 			e820_type = E820_TYPE_RESERVED;
166 			break;
167 		}
168 		e820__range_add(start, size, e820_type);
169 	}
170 	e820__update_table(e820_table);
171 }
172 
173 int __init efi_memblock_x86_reserve_range(void)
174 {
175 	struct efi_info *e = &boot_params.efi_info;
176 	struct efi_memory_map_data data;
177 	phys_addr_t pmap;
178 	int rv;
179 
180 	if (efi_enabled(EFI_PARAVIRT))
181 		return 0;
182 
183 #ifdef CONFIG_X86_32
184 	/* Can't handle data above 4GB at this time */
185 	if (e->efi_memmap_hi) {
186 		pr_err("Memory map is above 4GB, disabling EFI.\n");
187 		return -EINVAL;
188 	}
189 	pmap =  e->efi_memmap;
190 #else
191 	pmap = (e->efi_memmap |	((__u64)e->efi_memmap_hi << 32));
192 #endif
193 	data.phys_map		= pmap;
194 	data.size 		= e->efi_memmap_size;
195 	data.desc_size		= e->efi_memdesc_size;
196 	data.desc_version	= e->efi_memdesc_version;
197 
198 	rv = efi_memmap_init_early(&data);
199 	if (rv)
200 		return rv;
201 
202 	if (add_efi_memmap)
203 		do_add_efi_memmap();
204 
205 	WARN(efi.memmap.desc_version != 1,
206 	     "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
207 	     efi.memmap.desc_version);
208 
209 	memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
210 
211 	return 0;
212 }
213 
214 #define OVERFLOW_ADDR_SHIFT	(64 - EFI_PAGE_SHIFT)
215 #define OVERFLOW_ADDR_MASK	(U64_MAX << OVERFLOW_ADDR_SHIFT)
216 #define U64_HIGH_BIT		(~(U64_MAX >> 1))
217 
218 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
219 {
220 	u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
221 	u64 end_hi = 0;
222 	char buf[64];
223 
224 	if (md->num_pages == 0) {
225 		end = 0;
226 	} else if (md->num_pages > EFI_PAGES_MAX ||
227 		   EFI_PAGES_MAX - md->num_pages <
228 		   (md->phys_addr >> EFI_PAGE_SHIFT)) {
229 		end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
230 			>> OVERFLOW_ADDR_SHIFT;
231 
232 		if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
233 			end_hi += 1;
234 	} else {
235 		return true;
236 	}
237 
238 	pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
239 
240 	if (end_hi) {
241 		pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
242 			i, efi_md_typeattr_format(buf, sizeof(buf), md),
243 			md->phys_addr, end_hi, end);
244 	} else {
245 		pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
246 			i, efi_md_typeattr_format(buf, sizeof(buf), md),
247 			md->phys_addr, end);
248 	}
249 	return false;
250 }
251 
252 static void __init efi_clean_memmap(void)
253 {
254 	efi_memory_desc_t *out = efi.memmap.map;
255 	const efi_memory_desc_t *in = out;
256 	const efi_memory_desc_t *end = efi.memmap.map_end;
257 	int i, n_removal;
258 
259 	for (i = n_removal = 0; in < end; i++) {
260 		if (efi_memmap_entry_valid(in, i)) {
261 			if (out != in)
262 				memcpy(out, in, efi.memmap.desc_size);
263 			out = (void *)out + efi.memmap.desc_size;
264 		} else {
265 			n_removal++;
266 		}
267 		in = (void *)in + efi.memmap.desc_size;
268 	}
269 
270 	if (n_removal > 0) {
271 		u64 size = efi.memmap.nr_map - n_removal;
272 
273 		pr_warn("Removing %d invalid memory map entries.\n", n_removal);
274 		efi_memmap_install(efi.memmap.phys_map, size);
275 	}
276 }
277 
278 void __init efi_print_memmap(void)
279 {
280 	efi_memory_desc_t *md;
281 	int i = 0;
282 
283 	for_each_efi_memory_desc(md) {
284 		char buf[64];
285 
286 		pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
287 			i++, efi_md_typeattr_format(buf, sizeof(buf), md),
288 			md->phys_addr,
289 			md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
290 			(md->num_pages >> (20 - EFI_PAGE_SHIFT)));
291 	}
292 }
293 
294 static int __init efi_systab_init(void *phys)
295 {
296 	if (efi_enabled(EFI_64BIT)) {
297 		efi_system_table_64_t *systab64;
298 		struct efi_setup_data *data = NULL;
299 		u64 tmp = 0;
300 
301 		if (efi_setup) {
302 			data = early_memremap(efi_setup, sizeof(*data));
303 			if (!data)
304 				return -ENOMEM;
305 		}
306 		systab64 = early_memremap((unsigned long)phys,
307 					 sizeof(*systab64));
308 		if (systab64 == NULL) {
309 			pr_err("Couldn't map the system table!\n");
310 			if (data)
311 				early_memunmap(data, sizeof(*data));
312 			return -ENOMEM;
313 		}
314 
315 		efi_systab.hdr = systab64->hdr;
316 		efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
317 					      systab64->fw_vendor;
318 		tmp |= data ? data->fw_vendor : systab64->fw_vendor;
319 		efi_systab.fw_revision = systab64->fw_revision;
320 		efi_systab.con_in_handle = systab64->con_in_handle;
321 		tmp |= systab64->con_in_handle;
322 		efi_systab.con_in = systab64->con_in;
323 		tmp |= systab64->con_in;
324 		efi_systab.con_out_handle = systab64->con_out_handle;
325 		tmp |= systab64->con_out_handle;
326 		efi_systab.con_out = systab64->con_out;
327 		tmp |= systab64->con_out;
328 		efi_systab.stderr_handle = systab64->stderr_handle;
329 		tmp |= systab64->stderr_handle;
330 		efi_systab.stderr = systab64->stderr;
331 		tmp |= systab64->stderr;
332 		efi_systab.runtime = data ?
333 				     (void *)(unsigned long)data->runtime :
334 				     (void *)(unsigned long)systab64->runtime;
335 		tmp |= data ? data->runtime : systab64->runtime;
336 		efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
337 		tmp |= systab64->boottime;
338 		efi_systab.nr_tables = systab64->nr_tables;
339 		efi_systab.tables = data ? (unsigned long)data->tables :
340 					   systab64->tables;
341 		tmp |= data ? data->tables : systab64->tables;
342 
343 		early_memunmap(systab64, sizeof(*systab64));
344 		if (data)
345 			early_memunmap(data, sizeof(*data));
346 #ifdef CONFIG_X86_32
347 		if (tmp >> 32) {
348 			pr_err("EFI data located above 4GB, disabling EFI.\n");
349 			return -EINVAL;
350 		}
351 #endif
352 	} else {
353 		efi_system_table_32_t *systab32;
354 
355 		systab32 = early_memremap((unsigned long)phys,
356 					 sizeof(*systab32));
357 		if (systab32 == NULL) {
358 			pr_err("Couldn't map the system table!\n");
359 			return -ENOMEM;
360 		}
361 
362 		efi_systab.hdr = systab32->hdr;
363 		efi_systab.fw_vendor = systab32->fw_vendor;
364 		efi_systab.fw_revision = systab32->fw_revision;
365 		efi_systab.con_in_handle = systab32->con_in_handle;
366 		efi_systab.con_in = systab32->con_in;
367 		efi_systab.con_out_handle = systab32->con_out_handle;
368 		efi_systab.con_out = systab32->con_out;
369 		efi_systab.stderr_handle = systab32->stderr_handle;
370 		efi_systab.stderr = systab32->stderr;
371 		efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
372 		efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
373 		efi_systab.nr_tables = systab32->nr_tables;
374 		efi_systab.tables = systab32->tables;
375 
376 		early_memunmap(systab32, sizeof(*systab32));
377 	}
378 
379 	efi.systab = &efi_systab;
380 
381 	/*
382 	 * Verify the EFI Table
383 	 */
384 	if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
385 		pr_err("System table signature incorrect!\n");
386 		return -EINVAL;
387 	}
388 	if ((efi.systab->hdr.revision >> 16) == 0)
389 		pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
390 		       efi.systab->hdr.revision >> 16,
391 		       efi.systab->hdr.revision & 0xffff);
392 
393 	return 0;
394 }
395 
396 static int __init efi_runtime_init32(void)
397 {
398 	efi_runtime_services_32_t *runtime;
399 
400 	runtime = early_memremap((unsigned long)efi.systab->runtime,
401 			sizeof(efi_runtime_services_32_t));
402 	if (!runtime) {
403 		pr_err("Could not map the runtime service table!\n");
404 		return -ENOMEM;
405 	}
406 
407 	/*
408 	 * We will only need *early* access to the SetVirtualAddressMap
409 	 * EFI runtime service. All other runtime services will be called
410 	 * via the virtual mapping.
411 	 */
412 	efi_phys.set_virtual_address_map =
413 			(efi_set_virtual_address_map_t *)
414 			(unsigned long)runtime->set_virtual_address_map;
415 	early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
416 
417 	return 0;
418 }
419 
420 static int __init efi_runtime_init64(void)
421 {
422 	efi_runtime_services_64_t *runtime;
423 
424 	runtime = early_memremap((unsigned long)efi.systab->runtime,
425 			sizeof(efi_runtime_services_64_t));
426 	if (!runtime) {
427 		pr_err("Could not map the runtime service table!\n");
428 		return -ENOMEM;
429 	}
430 
431 	/*
432 	 * We will only need *early* access to the SetVirtualAddressMap
433 	 * EFI runtime service. All other runtime services will be called
434 	 * via the virtual mapping.
435 	 */
436 	efi_phys.set_virtual_address_map =
437 			(efi_set_virtual_address_map_t *)
438 			(unsigned long)runtime->set_virtual_address_map;
439 	early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
440 
441 	return 0;
442 }
443 
444 static int __init efi_runtime_init(void)
445 {
446 	int rv;
447 
448 	/*
449 	 * Check out the runtime services table. We need to map
450 	 * the runtime services table so that we can grab the physical
451 	 * address of several of the EFI runtime functions, needed to
452 	 * set the firmware into virtual mode.
453 	 *
454 	 * When EFI_PARAVIRT is in force then we could not map runtime
455 	 * service memory region because we do not have direct access to it.
456 	 * However, runtime services are available through proxy functions
457 	 * (e.g. in case of Xen dom0 EFI implementation they call special
458 	 * hypercall which executes relevant EFI functions) and that is why
459 	 * they are always enabled.
460 	 */
461 
462 	if (!efi_enabled(EFI_PARAVIRT)) {
463 		if (efi_enabled(EFI_64BIT))
464 			rv = efi_runtime_init64();
465 		else
466 			rv = efi_runtime_init32();
467 
468 		if (rv)
469 			return rv;
470 	}
471 
472 	set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
473 
474 	return 0;
475 }
476 
477 void __init efi_init(void)
478 {
479 	efi_char16_t *c16;
480 	char vendor[100] = "unknown";
481 	int i = 0;
482 	void *tmp;
483 
484 #ifdef CONFIG_X86_32
485 	if (boot_params.efi_info.efi_systab_hi ||
486 	    boot_params.efi_info.efi_memmap_hi) {
487 		pr_info("Table located above 4GB, disabling EFI.\n");
488 		return;
489 	}
490 	efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
491 #else
492 	efi_phys.systab = (efi_system_table_t *)
493 			  (boot_params.efi_info.efi_systab |
494 			  ((__u64)boot_params.efi_info.efi_systab_hi<<32));
495 #endif
496 
497 	if (efi_systab_init(efi_phys.systab))
498 		return;
499 
500 	efi.config_table = (unsigned long)efi.systab->tables;
501 	efi.fw_vendor	 = (unsigned long)efi.systab->fw_vendor;
502 	efi.runtime	 = (unsigned long)efi.systab->runtime;
503 
504 	/*
505 	 * Show what we know for posterity
506 	 */
507 	c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
508 	if (c16) {
509 		for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
510 			vendor[i] = *c16++;
511 		vendor[i] = '\0';
512 	} else
513 		pr_err("Could not map the firmware vendor!\n");
514 	early_memunmap(tmp, 2);
515 
516 	pr_info("EFI v%u.%.02u by %s\n",
517 		efi.systab->hdr.revision >> 16,
518 		efi.systab->hdr.revision & 0xffff, vendor);
519 
520 	if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
521 		return;
522 
523 	if (efi_config_init(arch_tables))
524 		return;
525 
526 	/*
527 	 * Note: We currently don't support runtime services on an EFI
528 	 * that doesn't match the kernel 32/64-bit mode.
529 	 */
530 
531 	if (!efi_runtime_supported())
532 		pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
533 	else {
534 		if (efi_runtime_disabled() || efi_runtime_init()) {
535 			efi_memmap_unmap();
536 			return;
537 		}
538 	}
539 
540 	efi_clean_memmap();
541 
542 	if (efi_enabled(EFI_DBG))
543 		efi_print_memmap();
544 }
545 
546 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
547 {
548 	u64 addr, npages;
549 
550 	addr = md->virt_addr;
551 	npages = md->num_pages;
552 
553 	memrange_efi_to_native(&addr, &npages);
554 
555 	if (executable)
556 		set_memory_x(addr, npages);
557 	else
558 		set_memory_nx(addr, npages);
559 }
560 
561 void __init runtime_code_page_mkexec(void)
562 {
563 	efi_memory_desc_t *md;
564 
565 	/* Make EFI runtime service code area executable */
566 	for_each_efi_memory_desc(md) {
567 		if (md->type != EFI_RUNTIME_SERVICES_CODE)
568 			continue;
569 
570 		efi_set_executable(md, true);
571 	}
572 }
573 
574 void __init efi_memory_uc(u64 addr, unsigned long size)
575 {
576 	unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
577 	u64 npages;
578 
579 	npages = round_up(size, page_shift) / page_shift;
580 	memrange_efi_to_native(&addr, &npages);
581 	set_memory_uc(addr, npages);
582 }
583 
584 void __init old_map_region(efi_memory_desc_t *md)
585 {
586 	u64 start_pfn, end_pfn, end;
587 	unsigned long size;
588 	void *va;
589 
590 	start_pfn = PFN_DOWN(md->phys_addr);
591 	size	  = md->num_pages << PAGE_SHIFT;
592 	end	  = md->phys_addr + size;
593 	end_pfn   = PFN_UP(end);
594 
595 	if (pfn_range_is_mapped(start_pfn, end_pfn)) {
596 		va = __va(md->phys_addr);
597 
598 		if (!(md->attribute & EFI_MEMORY_WB))
599 			efi_memory_uc((u64)(unsigned long)va, size);
600 	} else
601 		va = efi_ioremap(md->phys_addr, size,
602 				 md->type, md->attribute);
603 
604 	md->virt_addr = (u64) (unsigned long) va;
605 	if (!va)
606 		pr_err("ioremap of 0x%llX failed!\n",
607 		       (unsigned long long)md->phys_addr);
608 }
609 
610 /* Merge contiguous regions of the same type and attribute */
611 static void __init efi_merge_regions(void)
612 {
613 	efi_memory_desc_t *md, *prev_md = NULL;
614 
615 	for_each_efi_memory_desc(md) {
616 		u64 prev_size;
617 
618 		if (!prev_md) {
619 			prev_md = md;
620 			continue;
621 		}
622 
623 		if (prev_md->type != md->type ||
624 		    prev_md->attribute != md->attribute) {
625 			prev_md = md;
626 			continue;
627 		}
628 
629 		prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
630 
631 		if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
632 			prev_md->num_pages += md->num_pages;
633 			md->type = EFI_RESERVED_TYPE;
634 			md->attribute = 0;
635 			continue;
636 		}
637 		prev_md = md;
638 	}
639 }
640 
641 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
642 {
643 	unsigned long size;
644 	u64 end, systab;
645 
646 	size = md->num_pages << EFI_PAGE_SHIFT;
647 	end = md->phys_addr + size;
648 	systab = (u64)(unsigned long)efi_phys.systab;
649 	if (md->phys_addr <= systab && systab < end) {
650 		systab += md->virt_addr - md->phys_addr;
651 		efi.systab = (efi_system_table_t *)(unsigned long)systab;
652 	}
653 }
654 
655 static void *realloc_pages(void *old_memmap, int old_shift)
656 {
657 	void *ret;
658 
659 	ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
660 	if (!ret)
661 		goto out;
662 
663 	/*
664 	 * A first-time allocation doesn't have anything to copy.
665 	 */
666 	if (!old_memmap)
667 		return ret;
668 
669 	memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
670 
671 out:
672 	free_pages((unsigned long)old_memmap, old_shift);
673 	return ret;
674 }
675 
676 /*
677  * Iterate the EFI memory map in reverse order because the regions
678  * will be mapped top-down. The end result is the same as if we had
679  * mapped things forward, but doesn't require us to change the
680  * existing implementation of efi_map_region().
681  */
682 static inline void *efi_map_next_entry_reverse(void *entry)
683 {
684 	/* Initial call */
685 	if (!entry)
686 		return efi.memmap.map_end - efi.memmap.desc_size;
687 
688 	entry -= efi.memmap.desc_size;
689 	if (entry < efi.memmap.map)
690 		return NULL;
691 
692 	return entry;
693 }
694 
695 /*
696  * efi_map_next_entry - Return the next EFI memory map descriptor
697  * @entry: Previous EFI memory map descriptor
698  *
699  * This is a helper function to iterate over the EFI memory map, which
700  * we do in different orders depending on the current configuration.
701  *
702  * To begin traversing the memory map @entry must be %NULL.
703  *
704  * Returns %NULL when we reach the end of the memory map.
705  */
706 static void *efi_map_next_entry(void *entry)
707 {
708 	if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
709 		/*
710 		 * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
711 		 * config table feature requires us to map all entries
712 		 * in the same order as they appear in the EFI memory
713 		 * map. That is to say, entry N must have a lower
714 		 * virtual address than entry N+1. This is because the
715 		 * firmware toolchain leaves relative references in
716 		 * the code/data sections, which are split and become
717 		 * separate EFI memory regions. Mapping things
718 		 * out-of-order leads to the firmware accessing
719 		 * unmapped addresses.
720 		 *
721 		 * Since we need to map things this way whether or not
722 		 * the kernel actually makes use of
723 		 * EFI_PROPERTIES_TABLE, let's just switch to this
724 		 * scheme by default for 64-bit.
725 		 */
726 		return efi_map_next_entry_reverse(entry);
727 	}
728 
729 	/* Initial call */
730 	if (!entry)
731 		return efi.memmap.map;
732 
733 	entry += efi.memmap.desc_size;
734 	if (entry >= efi.memmap.map_end)
735 		return NULL;
736 
737 	return entry;
738 }
739 
740 static bool should_map_region(efi_memory_desc_t *md)
741 {
742 	/*
743 	 * Runtime regions always require runtime mappings (obviously).
744 	 */
745 	if (md->attribute & EFI_MEMORY_RUNTIME)
746 		return true;
747 
748 	/*
749 	 * 32-bit EFI doesn't suffer from the bug that requires us to
750 	 * reserve boot services regions, and mixed mode support
751 	 * doesn't exist for 32-bit kernels.
752 	 */
753 	if (IS_ENABLED(CONFIG_X86_32))
754 		return false;
755 
756 	/*
757 	 * Map all of RAM so that we can access arguments in the 1:1
758 	 * mapping when making EFI runtime calls.
759 	 */
760 	if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
761 		if (md->type == EFI_CONVENTIONAL_MEMORY ||
762 		    md->type == EFI_LOADER_DATA ||
763 		    md->type == EFI_LOADER_CODE)
764 			return true;
765 	}
766 
767 	/*
768 	 * Map boot services regions as a workaround for buggy
769 	 * firmware that accesses them even when they shouldn't.
770 	 *
771 	 * See efi_{reserve,free}_boot_services().
772 	 */
773 	if (md->type == EFI_BOOT_SERVICES_CODE ||
774 	    md->type == EFI_BOOT_SERVICES_DATA)
775 		return true;
776 
777 	return false;
778 }
779 
780 /*
781  * Map the efi memory ranges of the runtime services and update new_mmap with
782  * virtual addresses.
783  */
784 static void * __init efi_map_regions(int *count, int *pg_shift)
785 {
786 	void *p, *new_memmap = NULL;
787 	unsigned long left = 0;
788 	unsigned long desc_size;
789 	efi_memory_desc_t *md;
790 
791 	desc_size = efi.memmap.desc_size;
792 
793 	p = NULL;
794 	while ((p = efi_map_next_entry(p))) {
795 		md = p;
796 
797 		if (!should_map_region(md))
798 			continue;
799 
800 		efi_map_region(md);
801 		get_systab_virt_addr(md);
802 
803 		if (left < desc_size) {
804 			new_memmap = realloc_pages(new_memmap, *pg_shift);
805 			if (!new_memmap)
806 				return NULL;
807 
808 			left += PAGE_SIZE << *pg_shift;
809 			(*pg_shift)++;
810 		}
811 
812 		memcpy(new_memmap + (*count * desc_size), md, desc_size);
813 
814 		left -= desc_size;
815 		(*count)++;
816 	}
817 
818 	return new_memmap;
819 }
820 
821 static void __init kexec_enter_virtual_mode(void)
822 {
823 #ifdef CONFIG_KEXEC_CORE
824 	efi_memory_desc_t *md;
825 	unsigned int num_pages;
826 
827 	efi.systab = NULL;
828 
829 	/*
830 	 * We don't do virtual mode, since we don't do runtime services, on
831 	 * non-native EFI. With efi=old_map, we don't do runtime services in
832 	 * kexec kernel because in the initial boot something else might
833 	 * have been mapped at these virtual addresses.
834 	 */
835 	if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
836 		efi_memmap_unmap();
837 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
838 		return;
839 	}
840 
841 	if (efi_alloc_page_tables()) {
842 		pr_err("Failed to allocate EFI page tables\n");
843 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
844 		return;
845 	}
846 
847 	/*
848 	* Map efi regions which were passed via setup_data. The virt_addr is a
849 	* fixed addr which was used in first kernel of a kexec boot.
850 	*/
851 	for_each_efi_memory_desc(md) {
852 		efi_map_region_fixed(md); /* FIXME: add error handling */
853 		get_systab_virt_addr(md);
854 	}
855 
856 	/*
857 	 * Unregister the early EFI memmap from efi_init() and install
858 	 * the new EFI memory map.
859 	 */
860 	efi_memmap_unmap();
861 
862 	if (efi_memmap_init_late(efi.memmap.phys_map,
863 				 efi.memmap.desc_size * efi.memmap.nr_map)) {
864 		pr_err("Failed to remap late EFI memory map\n");
865 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
866 		return;
867 	}
868 
869 	BUG_ON(!efi.systab);
870 
871 	num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
872 	num_pages >>= PAGE_SHIFT;
873 
874 	if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
875 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
876 		return;
877 	}
878 
879 	efi_sync_low_kernel_mappings();
880 
881 	/*
882 	 * Now that EFI is in virtual mode, update the function
883 	 * pointers in the runtime service table to the new virtual addresses.
884 	 *
885 	 * Call EFI services through wrapper functions.
886 	 */
887 	efi.runtime_version = efi_systab.hdr.revision;
888 
889 	efi_native_runtime_setup();
890 
891 	efi.set_virtual_address_map = NULL;
892 
893 	if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
894 		runtime_code_page_mkexec();
895 
896 	/* clean DUMMY object */
897 	efi_delete_dummy_variable();
898 #endif
899 }
900 
901 /*
902  * This function will switch the EFI runtime services to virtual mode.
903  * Essentially, we look through the EFI memmap and map every region that
904  * has the runtime attribute bit set in its memory descriptor into the
905  * efi_pgd page table.
906  *
907  * The old method which used to update that memory descriptor with the
908  * virtual address obtained from ioremap() is still supported when the
909  * kernel is booted with efi=old_map on its command line. Same old
910  * method enabled the runtime services to be called without having to
911  * thunk back into physical mode for every invocation.
912  *
913  * The new method does a pagetable switch in a preemption-safe manner
914  * so that we're in a different address space when calling a runtime
915  * function. For function arguments passing we do copy the PUDs of the
916  * kernel page table into efi_pgd prior to each call.
917  *
918  * Specially for kexec boot, efi runtime maps in previous kernel should
919  * be passed in via setup_data. In that case runtime ranges will be mapped
920  * to the same virtual addresses as the first kernel, see
921  * kexec_enter_virtual_mode().
922  */
923 static void __init __efi_enter_virtual_mode(void)
924 {
925 	int count = 0, pg_shift = 0;
926 	void *new_memmap = NULL;
927 	efi_status_t status;
928 	unsigned long pa;
929 
930 	efi.systab = NULL;
931 
932 	if (efi_alloc_page_tables()) {
933 		pr_err("Failed to allocate EFI page tables\n");
934 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
935 		return;
936 	}
937 
938 	efi_merge_regions();
939 	new_memmap = efi_map_regions(&count, &pg_shift);
940 	if (!new_memmap) {
941 		pr_err("Error reallocating memory, EFI runtime non-functional!\n");
942 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
943 		return;
944 	}
945 
946 	pa = __pa(new_memmap);
947 
948 	/*
949 	 * Unregister the early EFI memmap from efi_init() and install
950 	 * the new EFI memory map that we are about to pass to the
951 	 * firmware via SetVirtualAddressMap().
952 	 */
953 	efi_memmap_unmap();
954 
955 	if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
956 		pr_err("Failed to remap late EFI memory map\n");
957 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
958 		return;
959 	}
960 
961 	if (efi_enabled(EFI_DBG)) {
962 		pr_info("EFI runtime memory map:\n");
963 		efi_print_memmap();
964 	}
965 
966 	BUG_ON(!efi.systab);
967 
968 	if (efi_setup_page_tables(pa, 1 << pg_shift)) {
969 		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
970 		return;
971 	}
972 
973 	efi_sync_low_kernel_mappings();
974 
975 	if (efi_is_native()) {
976 		status = phys_efi_set_virtual_address_map(
977 				efi.memmap.desc_size * count,
978 				efi.memmap.desc_size,
979 				efi.memmap.desc_version,
980 				(efi_memory_desc_t *)pa);
981 	} else {
982 		status = efi_thunk_set_virtual_address_map(
983 				efi_phys.set_virtual_address_map,
984 				efi.memmap.desc_size * count,
985 				efi.memmap.desc_size,
986 				efi.memmap.desc_version,
987 				(efi_memory_desc_t *)pa);
988 	}
989 
990 	if (status != EFI_SUCCESS) {
991 		pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
992 			 status);
993 		panic("EFI call to SetVirtualAddressMap() failed!");
994 	}
995 
996 	efi_free_boot_services();
997 
998 	/*
999 	 * Now that EFI is in virtual mode, update the function
1000 	 * pointers in the runtime service table to the new virtual addresses.
1001 	 *
1002 	 * Call EFI services through wrapper functions.
1003 	 */
1004 	efi.runtime_version = efi_systab.hdr.revision;
1005 
1006 	if (efi_is_native())
1007 		efi_native_runtime_setup();
1008 	else
1009 		efi_thunk_runtime_setup();
1010 
1011 	efi.set_virtual_address_map = NULL;
1012 
1013 	/*
1014 	 * Apply more restrictive page table mapping attributes now that
1015 	 * SVAM() has been called and the firmware has performed all
1016 	 * necessary relocation fixups for the new virtual addresses.
1017 	 */
1018 	efi_runtime_update_mappings();
1019 
1020 	/* clean DUMMY object */
1021 	efi_delete_dummy_variable();
1022 }
1023 
1024 void __init efi_enter_virtual_mode(void)
1025 {
1026 	if (efi_enabled(EFI_PARAVIRT))
1027 		return;
1028 
1029 	if (efi_setup)
1030 		kexec_enter_virtual_mode();
1031 	else
1032 		__efi_enter_virtual_mode();
1033 
1034 	efi_dump_pagetable();
1035 }
1036 
1037 static int __init arch_parse_efi_cmdline(char *str)
1038 {
1039 	if (!str) {
1040 		pr_warn("need at least one option\n");
1041 		return -EINVAL;
1042 	}
1043 
1044 	if (parse_option_str(str, "old_map"))
1045 		set_bit(EFI_OLD_MEMMAP, &efi.flags);
1046 
1047 	return 0;
1048 }
1049 early_param("efi", arch_parse_efi_cmdline);
1050