1 /*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * 4 * This software was developed by Benno Rice under sponsorship from 5 * the FreeBSD Foundation. 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 #include <sys/param.h> 30 31 #include <stand.h> 32 #include <bootstrap.h> 33 34 #include <efi.h> 35 #include <efilib.h> 36 37 #include "loader_efi.h" 38 39 #define M(x) ((x) * 1024 * 1024) 40 #define G(x) (1UL * (x) * 1024 * 1024 * 1024) 41 42 #if defined(__amd64__) 43 #include <machine/cpufunc.h> 44 #include <machine/specialreg.h> 45 #include <machine/vmparam.h> 46 47 /* 48 * The code is excerpted from sys/x86/x86/identcpu.c: identify_cpu(), 49 * identify_hypervisor(), and dev/hyperv/vmbus/hyperv.c: hyperv_identify(). 50 */ 51 #define CPUID_LEAF_HV_MAXLEAF 0x40000000 52 #define CPUID_LEAF_HV_INTERFACE 0x40000001 53 #define CPUID_LEAF_HV_FEATURES 0x40000003 54 #define CPUID_LEAF_HV_LIMITS 0x40000005 55 #define CPUID_HV_IFACE_HYPERV 0x31237648 /* HV#1 */ 56 #define CPUID_HV_MSR_TIME_REFCNT 0x0002 /* MSR_HV_TIME_REF_COUNT */ 57 #define CPUID_HV_MSR_HYPERCALL 0x0020 58 59 static int 60 running_on_hyperv(void) 61 { 62 char hv_vendor[16]; 63 uint32_t regs[4]; 64 65 do_cpuid(1, regs); 66 if ((regs[2] & CPUID2_HV) == 0) 67 return (0); 68 69 do_cpuid(CPUID_LEAF_HV_MAXLEAF, regs); 70 if (regs[0] < CPUID_LEAF_HV_LIMITS) 71 return (0); 72 73 ((uint32_t *)&hv_vendor)[0] = regs[1]; 74 ((uint32_t *)&hv_vendor)[1] = regs[2]; 75 ((uint32_t *)&hv_vendor)[2] = regs[3]; 76 hv_vendor[12] = '\0'; 77 if (strcmp(hv_vendor, "Microsoft Hv") != 0) 78 return (0); 79 80 do_cpuid(CPUID_LEAF_HV_INTERFACE, regs); 81 if (regs[0] != CPUID_HV_IFACE_HYPERV) 82 return (0); 83 84 do_cpuid(CPUID_LEAF_HV_FEATURES, regs); 85 if ((regs[0] & CPUID_HV_MSR_HYPERCALL) == 0) 86 return (0); 87 if ((regs[0] & CPUID_HV_MSR_TIME_REFCNT) == 0) 88 return (0); 89 90 return (1); 91 } 92 93 static void 94 efi_verify_staging_size(unsigned long *nr_pages) 95 { 96 UINTN sz; 97 EFI_MEMORY_DESCRIPTOR *map = NULL, *p; 98 EFI_PHYSICAL_ADDRESS start, end; 99 UINTN key, dsz; 100 UINT32 dver; 101 EFI_STATUS status; 102 int i, ndesc; 103 unsigned long available_pages = 0; 104 105 sz = 0; 106 107 for (;;) { 108 status = BS->GetMemoryMap(&sz, map, &key, &dsz, &dver); 109 if (!EFI_ERROR(status)) 110 break; 111 112 if (status != EFI_BUFFER_TOO_SMALL) { 113 printf("Can't read memory map: %lu\n", 114 EFI_ERROR_CODE(status)); 115 goto out; 116 } 117 118 free(map); 119 120 /* Allocate 10 descriptors more than the size reported, 121 * to allow for any fragmentation caused by calling 122 * malloc */ 123 map = malloc(sz + (10 * dsz)); 124 if (map == NULL) { 125 printf("Unable to allocate memory\n"); 126 goto out; 127 } 128 } 129 130 ndesc = sz / dsz; 131 for (i = 0, p = map; i < ndesc; 132 i++, p = NextMemoryDescriptor(p, dsz)) { 133 start = p->PhysicalStart; 134 end = start + p->NumberOfPages * EFI_PAGE_SIZE; 135 136 if (KERNLOAD < start || KERNLOAD >= end) 137 continue; 138 139 available_pages = p->NumberOfPages - 140 ((KERNLOAD - start) >> EFI_PAGE_SHIFT); 141 break; 142 } 143 144 if (available_pages == 0) { 145 printf("Can't find valid memory map for staging area!\n"); 146 goto out; 147 } 148 149 i++; 150 p = NextMemoryDescriptor(p, dsz); 151 152 for ( ; i < ndesc; 153 i++, p = NextMemoryDescriptor(p, dsz)) { 154 if (p->Type != EfiConventionalMemory && 155 p->Type != EfiLoaderData) 156 break; 157 158 if (p->PhysicalStart != end) 159 break; 160 161 end = p->PhysicalStart + p->NumberOfPages * EFI_PAGE_SIZE; 162 163 available_pages += p->NumberOfPages; 164 } 165 166 if (*nr_pages > available_pages) { 167 printf("Staging area's size is reduced: %ld -> %ld!\n", 168 *nr_pages, available_pages); 169 *nr_pages = available_pages; 170 } 171 out: 172 free(map); 173 } 174 #endif /* __amd64__ */ 175 176 #if defined(__arm__) 177 #define DEFAULT_EFI_STAGING_SIZE 32 178 #else 179 #define DEFAULT_EFI_STAGING_SIZE 64 180 #endif 181 #ifndef EFI_STAGING_SIZE 182 #define EFI_STAGING_SIZE DEFAULT_EFI_STAGING_SIZE 183 #endif 184 185 #if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ 186 defined(__riscv) 187 #define EFI_STAGING_2M_ALIGN 1 188 #else 189 #define EFI_STAGING_2M_ALIGN 0 190 #endif 191 192 #if defined(__amd64__) 193 #define EFI_STAGING_SLOP M(8) 194 #else 195 #define EFI_STAGING_SLOP 0 196 #endif 197 198 static u_long staging_slop = EFI_STAGING_SLOP; 199 200 EFI_PHYSICAL_ADDRESS staging, staging_end, staging_base; 201 bool stage_offset_set = false; 202 ssize_t stage_offset; 203 204 static void 205 efi_copy_free(void) 206 { 207 BS->FreePages(staging_base, (staging_end - staging_base) / 208 EFI_PAGE_SIZE); 209 stage_offset_set = false; 210 stage_offset = 0; 211 } 212 213 #ifdef __amd64__ 214 int copy_staging = COPY_STAGING_AUTO; 215 216 static int 217 command_copy_staging(int argc, char *argv[]) 218 { 219 static const char *const mode[3] = { 220 [COPY_STAGING_ENABLE] = "enable", 221 [COPY_STAGING_DISABLE] = "disable", 222 [COPY_STAGING_AUTO] = "auto", 223 }; 224 int prev, res; 225 226 res = CMD_OK; 227 if (argc > 2) { 228 res = CMD_ERROR; 229 } else if (argc == 2) { 230 prev = copy_staging; 231 if (strcmp(argv[1], "enable") == 0) 232 copy_staging = COPY_STAGING_ENABLE; 233 else if (strcmp(argv[1], "disable") == 0) 234 copy_staging = COPY_STAGING_DISABLE; 235 else if (strcmp(argv[1], "auto") == 0) 236 copy_staging = COPY_STAGING_AUTO; 237 else { 238 printf("usage: copy_staging enable|disable|auto\n"); 239 res = CMD_ERROR; 240 } 241 if (res == CMD_OK && prev != copy_staging) { 242 printf("changed copy_staging, unloading kernel\n"); 243 unload(); 244 efi_copy_free(); 245 efi_copy_init(); 246 } 247 } else { 248 printf("copy staging: %s\n", mode[copy_staging]); 249 } 250 return (res); 251 } 252 COMMAND_SET(copy_staging, "copy_staging", "copy staging", command_copy_staging); 253 #endif 254 255 static int 256 command_staging_slop(int argc, char *argv[]) 257 { 258 char *endp; 259 u_long new, prev; 260 int res; 261 262 res = CMD_OK; 263 if (argc > 2) { 264 res = CMD_ERROR; 265 } else if (argc == 2) { 266 new = strtoul(argv[1], &endp, 0); 267 if (*endp != '\0') { 268 printf("invalid slop value\n"); 269 res = CMD_ERROR; 270 } 271 if (res == CMD_OK && staging_slop != new) { 272 printf("changed slop, unloading kernel\n"); 273 unload(); 274 efi_copy_free(); 275 efi_copy_init(); 276 } 277 } else { 278 printf("staging slop %#lx\n", staging_slop); 279 } 280 return (res); 281 } 282 COMMAND_SET(staging_slop, "staging_slop", "set staging slop", 283 command_staging_slop); 284 285 #if defined(__amd64__) 286 /* 287 * The staging area must reside in the first 1GB or 4GB physical 288 * memory: see elf64_exec() in 289 * boot/efi/loader/arch/amd64/elf64_freebsd.c. 290 */ 291 static EFI_PHYSICAL_ADDRESS 292 get_staging_max(void) 293 { 294 EFI_PHYSICAL_ADDRESS res; 295 296 res = copy_staging == COPY_STAGING_ENABLE ? G(1) : G(4); 297 return (res); 298 } 299 #define EFI_ALLOC_METHOD AllocateMaxAddress 300 #else 301 #define EFI_ALLOC_METHOD AllocateAnyPages 302 #endif 303 304 int 305 efi_copy_init(void) 306 { 307 EFI_STATUS status; 308 unsigned long nr_pages; 309 vm_offset_t ess; 310 311 ess = EFI_STAGING_SIZE; 312 if (ess < DEFAULT_EFI_STAGING_SIZE) 313 ess = DEFAULT_EFI_STAGING_SIZE; 314 nr_pages = EFI_SIZE_TO_PAGES(M(1) * ess); 315 316 #if defined(__amd64__) 317 /* 318 * We'll decrease nr_pages, if it's too big. Currently we only 319 * apply this to FreeBSD VM running on Hyper-V. Why? Please see 320 * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=211746#c28 321 */ 322 if (running_on_hyperv()) 323 efi_verify_staging_size(&nr_pages); 324 325 staging = get_staging_max(); 326 #endif 327 status = BS->AllocatePages(EFI_ALLOC_METHOD, EfiLoaderCode, 328 nr_pages, &staging); 329 if (EFI_ERROR(status)) { 330 printf("failed to allocate staging area: %lu\n", 331 EFI_ERROR_CODE(status)); 332 return (status); 333 } 334 staging_base = staging; 335 staging_end = staging + nr_pages * EFI_PAGE_SIZE; 336 337 #if EFI_STAGING_2M_ALIGN 338 /* 339 * Round the kernel load address to a 2MiB value. This is needed 340 * because the kernel builds a page table based on where it has 341 * been loaded in physical address space. As the kernel will use 342 * either a 1MiB or 2MiB page for this we need to make sure it 343 * is correctly aligned for both cases. 344 */ 345 staging = roundup2(staging, M(2)); 346 #endif 347 348 return (0); 349 } 350 351 static bool 352 efi_check_space(vm_offset_t end) 353 { 354 EFI_PHYSICAL_ADDRESS addr, new_base, new_staging; 355 EFI_STATUS status; 356 unsigned long nr_pages; 357 358 end = roundup2(end, EFI_PAGE_SIZE); 359 360 /* There is already enough space */ 361 if (end + staging_slop <= staging_end) 362 return (true); 363 364 if (!boot_services_active) { 365 if (end <= staging_end) 366 return (true); 367 panic("efi_check_space: cannot expand staging area " 368 "after boot services were exited\n"); 369 } 370 371 /* 372 * Add slop at the end: 373 * 1. amd64 kernel expects to do some very early allocations 374 * by carving out memory after kernend. Slop guarantees 375 * that it does not ovewrite anything useful. 376 * 2. It seems that initial calculation of the staging size 377 * could be somewhat smaller than actually copying in after 378 * boot services are exited. Slop avoids calling 379 * BS->AllocatePages() when it cannot work. 380 */ 381 end += staging_slop; 382 383 nr_pages = EFI_SIZE_TO_PAGES(end - staging_end); 384 #if defined(__amd64__) 385 /* 386 * amd64 needs all memory to be allocated under the 1G or 4G boundary. 387 */ 388 if (end > get_staging_max()) 389 goto before_staging; 390 #endif 391 392 /* Try to allocate more space after the previous allocation */ 393 addr = staging_end; 394 status = BS->AllocatePages(AllocateAddress, EfiLoaderCode, nr_pages, 395 &addr); 396 if (!EFI_ERROR(status)) { 397 staging_end = staging_end + nr_pages * EFI_PAGE_SIZE; 398 return (true); 399 } 400 401 before_staging: 402 /* Try allocating space before the previous allocation */ 403 if (staging < nr_pages * EFI_PAGE_SIZE) 404 goto expand; 405 addr = staging - nr_pages * EFI_PAGE_SIZE; 406 #if EFI_STAGING_2M_ALIGN 407 /* See efi_copy_init for why this is needed */ 408 addr = rounddown2(addr, M(2)); 409 #endif 410 nr_pages = EFI_SIZE_TO_PAGES(staging_base - addr); 411 status = BS->AllocatePages(AllocateAddress, EfiLoaderCode, nr_pages, 412 &addr); 413 if (!EFI_ERROR(status)) { 414 /* 415 * Move the old allocation and update the state so 416 * translation still works. 417 */ 418 staging_base = addr; 419 memmove((void *)(uintptr_t)staging_base, 420 (void *)(uintptr_t)staging, staging_end - staging); 421 stage_offset -= staging - staging_base; 422 staging = staging_base; 423 return (true); 424 } 425 426 expand: 427 nr_pages = EFI_SIZE_TO_PAGES(end - (vm_offset_t)staging); 428 #if EFI_STAGING_2M_ALIGN 429 nr_pages += M(2) / EFI_PAGE_SIZE; 430 #endif 431 #if defined(__amd64__) 432 new_base = get_staging_max(); 433 #endif 434 status = BS->AllocatePages(EFI_ALLOC_METHOD, EfiLoaderCode, 435 nr_pages, &new_base); 436 if (!EFI_ERROR(status)) { 437 #if EFI_STAGING_2M_ALIGN 438 new_staging = roundup2(new_base, M(2)); 439 #else 440 new_staging = new_base; 441 #endif 442 /* 443 * Move the old allocation and update the state so 444 * translation still works. 445 */ 446 memcpy((void *)(uintptr_t)new_staging, 447 (void *)(uintptr_t)staging, staging_end - staging); 448 BS->FreePages(staging_base, (staging_end - staging_base) / 449 EFI_PAGE_SIZE); 450 stage_offset -= staging - new_staging; 451 staging = new_staging; 452 staging_end = new_base + nr_pages * EFI_PAGE_SIZE; 453 staging_base = new_base; 454 return (true); 455 } 456 457 printf("efi_check_space: Unable to expand staging area\n"); 458 return (false); 459 } 460 461 void * 462 efi_translate(vm_offset_t ptr) 463 { 464 465 return ((void *)(ptr + stage_offset)); 466 } 467 468 ssize_t 469 efi_copyin(const void *src, vm_offset_t dest, const size_t len) 470 { 471 472 if (!stage_offset_set) { 473 stage_offset = (vm_offset_t)staging - dest; 474 stage_offset_set = true; 475 } 476 477 /* XXX: Callers do not check for failure. */ 478 if (!efi_check_space(dest + stage_offset + len)) { 479 errno = ENOMEM; 480 return (-1); 481 } 482 bcopy(src, (void *)(dest + stage_offset), len); 483 return (len); 484 } 485 486 ssize_t 487 efi_copyout(const vm_offset_t src, void *dest, const size_t len) 488 { 489 490 /* XXX: Callers do not check for failure. */ 491 if (src + stage_offset + len > staging_end) { 492 errno = ENOMEM; 493 return (-1); 494 } 495 bcopy((void *)(src + stage_offset), dest, len); 496 return (len); 497 } 498 499 ssize_t 500 efi_readin(readin_handle_t fd, vm_offset_t dest, const size_t len) 501 { 502 503 if (!stage_offset_set) { 504 stage_offset = (vm_offset_t)staging - dest; 505 stage_offset_set = true; 506 } 507 508 if (!efi_check_space(dest + stage_offset + len)) { 509 errno = ENOMEM; 510 return (-1); 511 } 512 return (VECTX_READ(fd, (void *)(dest + stage_offset), len)); 513 } 514 515 void 516 efi_copy_finish(void) 517 { 518 uint64_t *src, *dst, *last; 519 520 src = (uint64_t *)(uintptr_t)staging; 521 dst = (uint64_t *)(uintptr_t)(staging - stage_offset); 522 last = (uint64_t *)(uintptr_t)staging_end; 523 524 while (src < last) 525 *dst++ = *src++; 526 } 527 528 void 529 efi_copy_finish_nop(void) 530 { 531 } 532