1 /*- 2 * Copyright (c) 2017 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by BAE Systems, the University of Cambridge 6 * Computer Laboratory, and Memorial University under DARPA/AFRL contract 7 * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing 8 * (TC) research program. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * Design overview. 34 * 35 * The driver provides character device for mmap(2) and ioctl(2) system calls 36 * allowing user to manage isolated compartments ("enclaves") in user VA space. 37 * 38 * The driver duties is EPC pages management, enclave management, user data 39 * validation. 40 * 41 * This driver requires Intel SGX support from hardware. 42 * 43 * /dev/sgx: 44 * .mmap: 45 * sgx_mmap_single() allocates VM object with following pager 46 * operations: 47 * a) sgx_pg_ctor(): 48 * VM object constructor does nothing 49 * b) sgx_pg_dtor(): 50 * VM object destructor destroys the SGX enclave associated 51 * with the object: it frees all the EPC pages allocated for 52 * enclave and removes the enclave. 53 * c) sgx_pg_fault(): 54 * VM object fault handler does nothing 55 * 56 * .ioctl: 57 * sgx_ioctl(): 58 * a) SGX_IOC_ENCLAVE_CREATE 59 * Adds Enclave SECS page: initial step of enclave creation. 60 * b) SGX_IOC_ENCLAVE_ADD_PAGE 61 * Adds TCS, REG pages to the enclave. 62 * c) SGX_IOC_ENCLAVE_INIT 63 * Finalizes enclave creation. 64 * 65 * Enclave lifecycle: 66 * .-- ECREATE -- Add SECS page 67 * Kernel | EADD -- Add TCS, REG pages 68 * space | EEXTEND -- Measure the page (take unique hash) 69 * ENCLS | EPA -- Allocate version array page 70 * '-- EINIT -- Finalize enclave creation 71 * User .-- EENTER -- Go to entry point of enclave 72 * space | EEXIT -- Exit back to main application 73 * ENCLU '-- ERESUME -- Resume enclave execution (e.g. after exception) 74 * 75 * Enclave lifecycle from driver point of view: 76 * 1) User calls mmap() on /dev/sgx: we allocate a VM object 77 * 2) User calls ioctl SGX_IOC_ENCLAVE_CREATE: we look for the VM object 78 * associated with user process created on step 1, create SECS physical 79 * page and store it in enclave's VM object queue by special index 80 * SGX_SECS_VM_OBJECT_INDEX. 81 * 3) User calls ioctl SGX_IOC_ENCLAVE_ADD_PAGE: we look for enclave created 82 * on step 2, create TCS or REG physical page and map it to specified by 83 * user address of enclave VM object. 84 * 4) User finalizes enclave creation with ioctl SGX_IOC_ENCLAVE_INIT call. 85 * 5) User can freely enter to and exit from enclave using ENCLU instructions 86 * from userspace: the driver does nothing here. 87 * 6) User proceed munmap(2) system call (or the process with enclave dies): 88 * we destroy the enclave associated with the object. 89 * 90 * EPC page types and their indexes in VM object queue: 91 * - PT_SECS index is special and equals SGX_SECS_VM_OBJECT_INDEX (-1); 92 * - PT_TCS and PT_REG indexes are specified by user in addr field of ioctl 93 * request data and determined as follows: 94 * pidx = OFF_TO_IDX(addp->addr - vmh->base); 95 * - PT_VA index is special, created for PT_REG, PT_TCS and PT_SECS pages 96 * and determined by formula: 97 * va_page_idx = - SGX_VA_PAGES_OFFS - (page_idx / SGX_VA_PAGE_SLOTS); 98 * PT_VA page can hold versions of up to 512 pages, and slot for each 99 * page in PT_VA page is determined as follows: 100 * va_slot_idx = page_idx % SGX_VA_PAGE_SLOTS; 101 * - PT_TRIM is unused. 102 * 103 * Locking: 104 * SGX ENCLS set of instructions have limitations on concurrency: 105 * some instructions can't be executed same time on different CPUs. 106 * We use sc->mtx_encls lock around them to prevent concurrent execution. 107 * sc->mtx lock is used to manage list of created enclaves and the state of 108 * SGX driver. 109 * 110 * Eviction of EPC pages: 111 * Eviction support is not implemented in this driver, however the driver 112 * manages VA (version array) pages: it allocates a VA slot for each EPC 113 * page. This will be required for eviction support in future. 114 * VA pages and slots are currently unused. 115 * 116 * Intel® 64 and IA-32 Architectures Software Developer's Manual 117 * https://software.intel.com/en-us/articles/intel-sdm 118 */ 119 120 #include <sys/param.h> 121 #include <sys/systm.h> 122 #include <sys/ioccom.h> 123 #include <sys/malloc.h> 124 #include <sys/kernel.h> 125 #include <sys/lock.h> 126 #include <sys/mutex.h> 127 #include <sys/rwlock.h> 128 #include <sys/conf.h> 129 #include <sys/module.h> 130 #include <sys/proc.h> 131 #include <sys/vmem.h> 132 #include <sys/vmmeter.h> 133 134 #include <vm/vm.h> 135 #include <vm/vm_param.h> 136 #include <vm/vm_extern.h> 137 #include <vm/vm_kern.h> 138 #include <vm/vm_page.h> 139 #include <vm/vm_map.h> 140 #include <vm/vm_object.h> 141 #include <vm/vm_pager.h> 142 #include <vm/vm_phys.h> 143 #include <vm/vm_radix.h> 144 #include <vm/pmap.h> 145 146 #include <machine/md_var.h> 147 #include <machine/specialreg.h> 148 #include <machine/cpufunc.h> 149 #include <machine/sgx.h> 150 #include <machine/sgxreg.h> 151 152 #include <amd64/sgx/sgxvar.h> 153 154 #define SGX_DEBUG 155 #undef SGX_DEBUG 156 157 #ifdef SGX_DEBUG 158 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) 159 #else 160 #define dprintf(fmt, ...) 161 #endif 162 163 static struct cdev_pager_ops sgx_pg_ops; 164 struct sgx_softc sgx_sc; 165 166 static int 167 sgx_get_epc_page(struct sgx_softc *sc, struct epc_page **epc) 168 { 169 vmem_addr_t addr; 170 int i; 171 172 if (vmem_alloc(sc->vmem_epc, PAGE_SIZE, M_FIRSTFIT | M_NOWAIT, 173 &addr) == 0) { 174 i = (addr - sc->epc_base) / PAGE_SIZE; 175 *epc = &sc->epc_pages[i]; 176 return (0); 177 } 178 179 return (ENOMEM); 180 } 181 182 static void 183 sgx_put_epc_page(struct sgx_softc *sc, struct epc_page *epc) 184 { 185 vmem_addr_t addr; 186 187 if (epc == NULL) 188 return; 189 190 addr = (epc->index * PAGE_SIZE) + sc->epc_base; 191 vmem_free(sc->vmem_epc, addr, PAGE_SIZE); 192 } 193 194 static void 195 sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object, 196 vm_pindex_t pidx, struct pctrie_iter *pages) 197 { 198 199 VM_OBJECT_ASSERT_WLOCKED(object); 200 201 page->valid = VM_PAGE_BITS_ALL; 202 vm_page_iter_insert(page, object, pidx, pages); 203 } 204 205 static int 206 sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object, 207 uint64_t idx, struct pctrie_iter *pages) 208 { 209 struct epc_page *epc; 210 vm_page_t page; 211 vm_page_t p; 212 int ret; 213 214 VM_OBJECT_ASSERT_WLOCKED(object); 215 216 p = vm_radix_iter_lookup(pages, idx); 217 if (p == NULL) { 218 ret = sgx_get_epc_page(sc, &epc); 219 if (ret) { 220 dprintf("%s: No free EPC pages available.\n", 221 __func__); 222 return (ret); 223 } 224 225 mtx_lock(&sc->mtx_encls); 226 sgx_epa((void *)epc->base); 227 mtx_unlock(&sc->mtx_encls); 228 229 page = PHYS_TO_VM_PAGE(epc->phys); 230 sgx_insert_epc_page_by_index(page, object, idx, pages); 231 } 232 233 return (0); 234 } 235 236 static int 237 sgx_va_slot_init(struct sgx_softc *sc, struct sgx_enclave *enclave, 238 vm_pindex_t pidx, struct pctrie_iter *pages) 239 { 240 uint64_t va_page_idx; 241 uint64_t idx; 242 vm_object_t object; 243 int ret; 244 245 object = enclave->object; 246 247 VM_OBJECT_ASSERT_WLOCKED(object); 248 249 va_page_idx = pidx / SGX_VA_PAGE_SLOTS; 250 idx = - SGX_VA_PAGES_OFFS - va_page_idx; 251 252 ret = sgx_va_slot_init_by_index(sc, object, idx, pages); 253 254 return (ret); 255 } 256 257 static int 258 sgx_mem_find(struct sgx_softc *sc, uint64_t addr, 259 vm_map_entry_t *entry0, vm_object_t *object0) 260 { 261 vm_map_t map; 262 vm_map_entry_t entry; 263 vm_object_t object; 264 265 map = &curproc->p_vmspace->vm_map; 266 267 vm_map_lock_read(map); 268 if (!vm_map_lookup_entry(map, addr, &entry)) { 269 vm_map_unlock_read(map); 270 dprintf("%s: Can't find enclave.\n", __func__); 271 return (EINVAL); 272 } 273 274 object = entry->object.vm_object; 275 if (object == NULL || object->handle == NULL) { 276 vm_map_unlock_read(map); 277 return (EINVAL); 278 } 279 280 if (object->type != OBJT_MGTDEVICE || 281 object->un_pager.devp.ops != &sgx_pg_ops) { 282 vm_map_unlock_read(map); 283 return (EINVAL); 284 } 285 286 vm_object_reference(object); 287 288 *object0 = object; 289 *entry0 = entry; 290 vm_map_unlock_read(map); 291 292 return (0); 293 } 294 295 static int 296 sgx_enclave_find(struct sgx_softc *sc, uint64_t addr, 297 struct sgx_enclave **encl) 298 { 299 struct sgx_vm_handle *vmh; 300 struct sgx_enclave *enclave; 301 vm_map_entry_t entry; 302 vm_object_t object; 303 int ret; 304 305 ret = sgx_mem_find(sc, addr, &entry, &object); 306 if (ret) 307 return (ret); 308 309 vmh = object->handle; 310 if (vmh == NULL) { 311 vm_object_deallocate(object); 312 return (EINVAL); 313 } 314 315 enclave = vmh->enclave; 316 if (enclave == NULL || enclave->object == NULL) { 317 vm_object_deallocate(object); 318 return (EINVAL); 319 } 320 321 *encl = enclave; 322 323 return (0); 324 } 325 326 static int 327 sgx_enclave_alloc(struct sgx_softc *sc, struct secs *secs, 328 struct sgx_enclave **enclave0) 329 { 330 struct sgx_enclave *enclave; 331 332 enclave = malloc(sizeof(struct sgx_enclave), 333 M_SGX, M_WAITOK | M_ZERO); 334 335 enclave->base = secs->base; 336 enclave->size = secs->size; 337 338 *enclave0 = enclave; 339 340 return (0); 341 } 342 343 static void 344 sgx_epc_page_remove(struct sgx_softc *sc, 345 struct epc_page *epc) 346 { 347 348 mtx_lock(&sc->mtx_encls); 349 sgx_eremove((void *)epc->base); 350 mtx_unlock(&sc->mtx_encls); 351 } 352 353 static void 354 sgx_page_remove(struct sgx_softc *sc, vm_page_t p, 355 struct pctrie_iter *pages) 356 { 357 struct epc_page *epc; 358 vm_paddr_t pa; 359 uint64_t offs; 360 361 if (pages != NULL) 362 (void)vm_page_iter_remove(pages, p); 363 else 364 (void) vm_page_remove(p); 365 366 dprintf("%s: p->pidx %ld\n", __func__, p->pindex); 367 368 pa = VM_PAGE_TO_PHYS(p); 369 epc = &sc->epc_pages[0]; 370 offs = (pa - epc->phys) / PAGE_SIZE; 371 epc = &sc->epc_pages[offs]; 372 373 sgx_epc_page_remove(sc, epc); 374 sgx_put_epc_page(sc, epc); 375 } 376 377 static void 378 sgx_enclave_remove(struct sgx_softc *sc, 379 struct sgx_enclave *enclave) 380 { 381 struct pctrie_iter pages; 382 vm_object_t object; 383 vm_page_t p, p_secs; 384 385 mtx_lock(&sc->mtx); 386 TAILQ_REMOVE(&sc->enclaves, enclave, next); 387 mtx_unlock(&sc->mtx); 388 389 object = enclave->object; 390 391 vm_page_iter_init(&pages, object); 392 VM_OBJECT_WLOCK(object); 393 394 /* 395 * First remove all the pages except SECS, 396 * then remove SECS page. 397 */ 398 restart: 399 VM_RADIX_FOREACH(p, &pages) { 400 if (p->pindex == SGX_SECS_VM_OBJECT_INDEX) 401 continue; 402 if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) { 403 pctrie_iter_reset(&pages); 404 goto restart; 405 } 406 sgx_page_remove(sc, p, &pages); 407 } 408 p_secs = vm_page_grab(object, SGX_SECS_VM_OBJECT_INDEX, 409 VM_ALLOC_NOCREAT); 410 /* Now remove SECS page */ 411 if (p_secs != NULL) 412 sgx_page_remove(sc, p_secs, NULL); 413 414 KASSERT(object->resident_page_count == 0, ("count")); 415 416 VM_OBJECT_WUNLOCK(object); 417 } 418 419 static int 420 sgx_measure_page(struct sgx_softc *sc, struct epc_page *secs, 421 struct epc_page *epc, uint16_t mrmask) 422 { 423 int i, j; 424 int ret; 425 426 mtx_lock(&sc->mtx_encls); 427 428 for (i = 0, j = 1; i < PAGE_SIZE; i += 0x100, j <<= 1) { 429 if (!(j & mrmask)) 430 continue; 431 432 ret = sgx_eextend((void *)secs->base, 433 (void *)(epc->base + i)); 434 if (ret == SGX_EFAULT) { 435 mtx_unlock(&sc->mtx_encls); 436 return (ret); 437 } 438 } 439 440 mtx_unlock(&sc->mtx_encls); 441 442 return (0); 443 } 444 445 static int 446 sgx_secs_validate(struct sgx_softc *sc, struct secs *secs) 447 { 448 struct secs_attr *attr; 449 int i; 450 451 if (secs->size == 0) 452 return (EINVAL); 453 454 /* BASEADDR must be naturally aligned on an SECS.SIZE boundary. */ 455 if (secs->base & (secs->size - 1)) 456 return (EINVAL); 457 458 /* SECS.SIZE must be at least 2 pages. */ 459 if (secs->size < 2 * PAGE_SIZE) 460 return (EINVAL); 461 462 if ((secs->size & (secs->size - 1)) != 0) 463 return (EINVAL); 464 465 attr = &secs->attributes; 466 467 if (attr->reserved1 != 0 || 468 attr->reserved2 != 0 || 469 attr->reserved3 != 0) 470 return (EINVAL); 471 472 for (i = 0; i < SECS_ATTR_RSV4_SIZE; i++) 473 if (attr->reserved4[i]) 474 return (EINVAL); 475 476 /* 477 * Intel® Software Guard Extensions Programming Reference 478 * 6.7.2 Relevant Fields in Various Data Structures 479 * 6.7.2.1 SECS.ATTRIBUTES.XFRM 480 * XFRM[1:0] must be set to 0x3. 481 */ 482 if ((attr->xfrm & 0x3) != 0x3) 483 return (EINVAL); 484 485 if (!attr->mode64bit) 486 return (EINVAL); 487 488 if (secs->size > sc->enclave_size_max) 489 return (EINVAL); 490 491 for (i = 0; i < SECS_RSV1_SIZE; i++) 492 if (secs->reserved1[i]) 493 return (EINVAL); 494 495 for (i = 0; i < SECS_RSV2_SIZE; i++) 496 if (secs->reserved2[i]) 497 return (EINVAL); 498 499 for (i = 0; i < SECS_RSV3_SIZE; i++) 500 if (secs->reserved3[i]) 501 return (EINVAL); 502 503 for (i = 0; i < SECS_RSV4_SIZE; i++) 504 if (secs->reserved4[i]) 505 return (EINVAL); 506 507 return (0); 508 } 509 510 static int 511 sgx_tcs_validate(struct tcs *tcs) 512 { 513 int i; 514 515 if ((tcs->flags) || 516 (tcs->ossa & (PAGE_SIZE - 1)) || 517 (tcs->ofsbasgx & (PAGE_SIZE - 1)) || 518 (tcs->ogsbasgx & (PAGE_SIZE - 1)) || 519 ((tcs->fslimit & 0xfff) != 0xfff) || 520 ((tcs->gslimit & 0xfff) != 0xfff)) 521 return (EINVAL); 522 523 for (i = 0; i < nitems(tcs->reserved3); i++) 524 if (tcs->reserved3[i]) 525 return (EINVAL); 526 527 return (0); 528 } 529 530 static void 531 sgx_tcs_dump(struct sgx_softc *sc, struct tcs *t) 532 { 533 534 dprintf("t->flags %lx\n", t->flags); 535 dprintf("t->ossa %lx\n", t->ossa); 536 dprintf("t->cssa %x\n", t->cssa); 537 dprintf("t->nssa %x\n", t->nssa); 538 dprintf("t->oentry %lx\n", t->oentry); 539 dprintf("t->ofsbasgx %lx\n", t->ofsbasgx); 540 dprintf("t->ogsbasgx %lx\n", t->ogsbasgx); 541 dprintf("t->fslimit %x\n", t->fslimit); 542 dprintf("t->gslimit %x\n", t->gslimit); 543 } 544 545 static int 546 sgx_pg_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 547 vm_ooffset_t foff, struct ucred *cred, u_short *color) 548 { 549 struct sgx_vm_handle *vmh; 550 551 vmh = handle; 552 if (vmh == NULL) { 553 dprintf("%s: vmh not found.\n", __func__); 554 return (0); 555 } 556 557 dprintf("%s: vmh->base %lx foff 0x%lx size 0x%lx\n", 558 __func__, vmh->base, foff, size); 559 560 return (0); 561 } 562 563 static void 564 sgx_pg_dtor(void *handle) 565 { 566 struct sgx_vm_handle *vmh; 567 struct sgx_softc *sc; 568 569 vmh = handle; 570 if (vmh == NULL) { 571 dprintf("%s: vmh not found.\n", __func__); 572 return; 573 } 574 575 sc = vmh->sc; 576 if (sc == NULL) { 577 dprintf("%s: sc is NULL\n", __func__); 578 return; 579 } 580 581 if (vmh->enclave == NULL) { 582 dprintf("%s: Enclave not found.\n", __func__); 583 return; 584 } 585 586 sgx_enclave_remove(sc, vmh->enclave); 587 588 free(vmh->enclave, M_SGX); 589 free(vmh, M_SGX); 590 } 591 592 static int 593 sgx_pg_fault(vm_object_t object, vm_ooffset_t offset, 594 int prot, vm_page_t *mres) 595 { 596 597 /* 598 * The purpose of this trivial handler is to handle the race 599 * when user tries to access mmaped region before or during 600 * enclave creation ioctl calls. 601 */ 602 603 dprintf("%s: offset 0x%lx\n", __func__, offset); 604 605 return (VM_PAGER_FAIL); 606 } 607 608 static void 609 sgx_pg_path(void *handle, char *path, size_t len) 610 { 611 strlcpy(path, "sgx", len); 612 } 613 614 static struct cdev_pager_ops sgx_pg_ops = { 615 .cdev_pg_ctor = sgx_pg_ctor, 616 .cdev_pg_dtor = sgx_pg_dtor, 617 .cdev_pg_fault = sgx_pg_fault, 618 .cdev_pg_path = sgx_pg_path, 619 }; 620 621 static void 622 sgx_insert_epc_page(struct sgx_enclave *enclave, struct epc_page *epc, 623 uint64_t addr, struct pctrie_iter *pages) 624 { 625 vm_pindex_t pidx; 626 vm_page_t page; 627 628 VM_OBJECT_ASSERT_WLOCKED(enclave->object); 629 630 pidx = OFF_TO_IDX(addr); 631 page = PHYS_TO_VM_PAGE(epc->phys); 632 633 sgx_insert_epc_page_by_index(page, enclave->object, pidx, pages); 634 } 635 636 static int 637 sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param) 638 { 639 struct pctrie_iter pages; 640 struct sgx_vm_handle *vmh; 641 vm_map_entry_t entry; 642 vm_page_t p; 643 struct page_info pginfo; 644 struct secinfo secinfo; 645 struct sgx_enclave *enclave; 646 struct epc_page *epc; 647 struct secs *secs; 648 vm_object_t object; 649 vm_page_t page; 650 int ret; 651 652 epc = NULL; 653 secs = NULL; 654 enclave = NULL; 655 object = NULL; 656 657 /* SGX Enclave Control Structure (SECS) */ 658 secs = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO); 659 ret = copyin((void *)param->src, secs, sizeof(struct secs)); 660 if (ret) { 661 dprintf("%s: Can't copy SECS.\n", __func__); 662 goto error; 663 } 664 665 ret = sgx_secs_validate(sc, secs); 666 if (ret) { 667 dprintf("%s: SECS validation failed.\n", __func__); 668 goto error; 669 } 670 671 ret = sgx_mem_find(sc, secs->base, &entry, &object); 672 if (ret) { 673 dprintf("%s: Can't find vm_map.\n", __func__); 674 goto error; 675 } 676 677 vmh = object->handle; 678 if (!vmh) { 679 dprintf("%s: Can't find vmh.\n", __func__); 680 ret = ENXIO; 681 goto error; 682 } 683 684 dprintf("%s: entry start %lx offset %lx\n", 685 __func__, entry->start, entry->offset); 686 vmh->base = (entry->start - entry->offset); 687 688 ret = sgx_enclave_alloc(sc, secs, &enclave); 689 if (ret) { 690 dprintf("%s: Can't alloc enclave.\n", __func__); 691 goto error; 692 } 693 enclave->object = object; 694 enclave->vmh = vmh; 695 696 memset(&secinfo, 0, sizeof(struct secinfo)); 697 memset(&pginfo, 0, sizeof(struct page_info)); 698 pginfo.linaddr = 0; 699 pginfo.srcpge = (uint64_t)secs; 700 pginfo.secinfo = &secinfo; 701 pginfo.secs = 0; 702 703 ret = sgx_get_epc_page(sc, &epc); 704 if (ret) { 705 dprintf("%s: Failed to get free epc page.\n", __func__); 706 goto error; 707 } 708 enclave->secs_epc_page = epc; 709 710 vm_page_iter_init(&pages, object); 711 VM_OBJECT_WLOCK(object); 712 p = vm_radix_iter_lookup(&pages, SGX_SECS_VM_OBJECT_INDEX); 713 if (p) { 714 VM_OBJECT_WUNLOCK(object); 715 /* SECS page already added. */ 716 ret = ENXIO; 717 goto error; 718 } 719 720 ret = sgx_va_slot_init_by_index(sc, object, 721 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, &pages); 722 if (ret) { 723 VM_OBJECT_WUNLOCK(object); 724 dprintf("%s: Can't init va slot.\n", __func__); 725 goto error; 726 } 727 728 mtx_lock(&sc->mtx); 729 if ((sc->state & SGX_STATE_RUNNING) == 0) { 730 mtx_unlock(&sc->mtx); 731 /* Remove VA page that was just created for SECS page. */ 732 p = vm_page_grab(enclave->object, 733 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, 734 VM_ALLOC_NOCREAT); 735 sgx_page_remove(sc, p, NULL); 736 VM_OBJECT_WUNLOCK(object); 737 goto error; 738 } 739 mtx_lock(&sc->mtx_encls); 740 ret = sgx_ecreate(&pginfo, (void *)epc->base); 741 mtx_unlock(&sc->mtx_encls); 742 if (ret == SGX_EFAULT) { 743 dprintf("%s: gp fault\n", __func__); 744 mtx_unlock(&sc->mtx); 745 /* Remove VA page that was just created for SECS page. */ 746 p = vm_page_grab(enclave->object, 747 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, 748 VM_ALLOC_NOCREAT); 749 sgx_page_remove(sc, p, NULL); 750 VM_OBJECT_WUNLOCK(object); 751 goto error; 752 } 753 754 TAILQ_INSERT_TAIL(&sc->enclaves, enclave, next); 755 mtx_unlock(&sc->mtx); 756 757 vmh->enclave = enclave; 758 759 page = PHYS_TO_VM_PAGE(epc->phys); 760 sgx_insert_epc_page_by_index(page, enclave->object, 761 SGX_SECS_VM_OBJECT_INDEX, &pages); 762 763 VM_OBJECT_WUNLOCK(object); 764 765 /* Release the reference. */ 766 vm_object_deallocate(object); 767 768 free(secs, M_SGX); 769 770 return (0); 771 772 error: 773 free(secs, M_SGX); 774 sgx_put_epc_page(sc, epc); 775 free(enclave, M_SGX); 776 vm_object_deallocate(object); 777 778 return (ret); 779 } 780 781 static int 782 sgx_ioctl_add_page(struct sgx_softc *sc, 783 struct sgx_enclave_add_page *addp) 784 { 785 struct pctrie_iter pages; 786 struct epc_page *secs_epc_page; 787 struct sgx_enclave *enclave; 788 struct sgx_vm_handle *vmh; 789 struct epc_page *epc; 790 struct page_info pginfo; 791 struct secinfo secinfo; 792 vm_object_t object; 793 void *tmp_vaddr; 794 uint64_t page_type; 795 struct tcs *t; 796 uint64_t addr; 797 uint64_t pidx; 798 vm_page_t p; 799 int ret; 800 801 tmp_vaddr = NULL; 802 epc = NULL; 803 object = NULL; 804 805 /* Find and get reference to VM object. */ 806 ret = sgx_enclave_find(sc, addp->addr, &enclave); 807 if (ret) { 808 dprintf("%s: Failed to find enclave.\n", __func__); 809 goto error; 810 } 811 812 object = enclave->object; 813 KASSERT(object != NULL, ("vm object is NULL\n")); 814 vmh = object->handle; 815 816 ret = sgx_get_epc_page(sc, &epc); 817 if (ret) { 818 dprintf("%s: Failed to get free epc page.\n", __func__); 819 goto error; 820 } 821 822 memset(&secinfo, 0, sizeof(struct secinfo)); 823 ret = copyin((void *)addp->secinfo, &secinfo, 824 sizeof(struct secinfo)); 825 if (ret) { 826 dprintf("%s: Failed to copy secinfo.\n", __func__); 827 goto error; 828 } 829 830 tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO); 831 ret = copyin((void *)addp->src, tmp_vaddr, PAGE_SIZE); 832 if (ret) { 833 dprintf("%s: Failed to copy page.\n", __func__); 834 goto error; 835 } 836 837 page_type = (secinfo.flags & SECINFO_FLAGS_PT_M) >> 838 SECINFO_FLAGS_PT_S; 839 if (page_type != SGX_PT_TCS && page_type != SGX_PT_REG) { 840 dprintf("%s: page can't be added.\n", __func__); 841 goto error; 842 } 843 if (page_type == SGX_PT_TCS) { 844 t = (struct tcs *)tmp_vaddr; 845 ret = sgx_tcs_validate(t); 846 if (ret) { 847 dprintf("%s: TCS page validation failed.\n", 848 __func__); 849 goto error; 850 } 851 sgx_tcs_dump(sc, t); 852 } 853 854 addr = (addp->addr - vmh->base); 855 pidx = OFF_TO_IDX(addr); 856 857 vm_page_iter_init(&pages, object); 858 VM_OBJECT_WLOCK(object); 859 p = vm_radix_iter_lookup(&pages, pidx); 860 if (p) { 861 VM_OBJECT_WUNLOCK(object); 862 /* Page already added. */ 863 ret = ENXIO; 864 goto error; 865 } 866 867 ret = sgx_va_slot_init(sc, enclave, pidx, &pages); 868 if (ret) { 869 VM_OBJECT_WUNLOCK(object); 870 dprintf("%s: Can't init va slot.\n", __func__); 871 goto error; 872 } 873 874 secs_epc_page = enclave->secs_epc_page; 875 memset(&pginfo, 0, sizeof(struct page_info)); 876 pginfo.linaddr = (uint64_t)addp->addr; 877 pginfo.srcpge = (uint64_t)tmp_vaddr; 878 pginfo.secinfo = &secinfo; 879 pginfo.secs = (uint64_t)secs_epc_page->base; 880 881 mtx_lock(&sc->mtx_encls); 882 ret = sgx_eadd(&pginfo, (void *)epc->base); 883 if (ret == SGX_EFAULT) { 884 dprintf("%s: gp fault on eadd\n", __func__); 885 mtx_unlock(&sc->mtx_encls); 886 VM_OBJECT_WUNLOCK(object); 887 goto error; 888 } 889 mtx_unlock(&sc->mtx_encls); 890 891 ret = sgx_measure_page(sc, enclave->secs_epc_page, epc, addp->mrmask); 892 if (ret == SGX_EFAULT) { 893 dprintf("%s: gp fault on eextend\n", __func__); 894 sgx_epc_page_remove(sc, epc); 895 VM_OBJECT_WUNLOCK(object); 896 goto error; 897 } 898 899 sgx_insert_epc_page(enclave, epc, addr, &pages); 900 901 VM_OBJECT_WUNLOCK(object); 902 903 /* Release the reference. */ 904 vm_object_deallocate(object); 905 906 free(tmp_vaddr, M_SGX); 907 908 return (0); 909 910 error: 911 free(tmp_vaddr, M_SGX); 912 sgx_put_epc_page(sc, epc); 913 vm_object_deallocate(object); 914 915 return (ret); 916 } 917 918 static int 919 sgx_ioctl_init(struct sgx_softc *sc, struct sgx_enclave_init *initp) 920 { 921 struct epc_page *secs_epc_page; 922 struct sgx_enclave *enclave; 923 struct thread *td; 924 void *tmp_vaddr; 925 void *einittoken; 926 void *sigstruct; 927 vm_object_t object; 928 int retry; 929 int ret; 930 931 td = curthread; 932 tmp_vaddr = NULL; 933 object = NULL; 934 935 dprintf("%s: addr %lx, sigstruct %lx, einittoken %lx\n", 936 __func__, initp->addr, initp->sigstruct, initp->einittoken); 937 938 /* Find and get reference to VM object. */ 939 ret = sgx_enclave_find(sc, initp->addr, &enclave); 940 if (ret) { 941 dprintf("%s: Failed to find enclave.\n", __func__); 942 goto error; 943 } 944 945 object = enclave->object; 946 947 tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO); 948 sigstruct = tmp_vaddr; 949 einittoken = (void *)((uint64_t)sigstruct + PAGE_SIZE / 2); 950 951 ret = copyin((void *)initp->sigstruct, sigstruct, 952 SGX_SIGSTRUCT_SIZE); 953 if (ret) { 954 dprintf("%s: Failed to copy SIGSTRUCT page.\n", __func__); 955 goto error; 956 } 957 958 ret = copyin((void *)initp->einittoken, einittoken, 959 SGX_EINITTOKEN_SIZE); 960 if (ret) { 961 dprintf("%s: Failed to copy EINITTOKEN page.\n", __func__); 962 goto error; 963 } 964 965 secs_epc_page = enclave->secs_epc_page; 966 retry = 16; 967 do { 968 mtx_lock(&sc->mtx_encls); 969 ret = sgx_einit(sigstruct, (void *)secs_epc_page->base, 970 einittoken); 971 mtx_unlock(&sc->mtx_encls); 972 dprintf("%s: sgx_einit returned %d\n", __func__, ret); 973 } while (ret == SGX_UNMASKED_EVENT && retry--); 974 975 if (ret) { 976 dprintf("%s: Failed init enclave: %d\n", __func__, ret); 977 td->td_retval[0] = ret; 978 ret = 0; 979 } 980 981 error: 982 free(tmp_vaddr, M_SGX); 983 984 /* Release the reference. */ 985 vm_object_deallocate(object); 986 987 return (ret); 988 } 989 990 static int 991 sgx_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, 992 struct thread *td) 993 { 994 struct sgx_enclave_add_page *addp; 995 struct sgx_enclave_create *param; 996 struct sgx_enclave_init *initp; 997 struct sgx_softc *sc; 998 int ret; 999 int len; 1000 1001 sc = &sgx_sc; 1002 1003 len = IOCPARM_LEN(cmd); 1004 1005 dprintf("%s: cmd %lx, addr %lx, len %d\n", 1006 __func__, cmd, (uint64_t)addr, len); 1007 1008 if (len > SGX_IOCTL_MAX_DATA_LEN) 1009 return (EINVAL); 1010 1011 switch (cmd) { 1012 case SGX_IOC_ENCLAVE_CREATE: 1013 param = (struct sgx_enclave_create *)addr; 1014 ret = sgx_ioctl_create(sc, param); 1015 break; 1016 case SGX_IOC_ENCLAVE_ADD_PAGE: 1017 addp = (struct sgx_enclave_add_page *)addr; 1018 ret = sgx_ioctl_add_page(sc, addp); 1019 break; 1020 case SGX_IOC_ENCLAVE_INIT: 1021 initp = (struct sgx_enclave_init *)addr; 1022 ret = sgx_ioctl_init(sc, initp); 1023 break; 1024 default: 1025 return (EINVAL); 1026 } 1027 1028 return (ret); 1029 } 1030 1031 static int 1032 sgx_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, 1033 vm_size_t mapsize, struct vm_object **objp, int nprot) 1034 { 1035 struct sgx_vm_handle *vmh; 1036 struct sgx_softc *sc; 1037 1038 sc = &sgx_sc; 1039 1040 dprintf("%s: mapsize 0x%lx, offset %lx\n", 1041 __func__, mapsize, *offset); 1042 1043 vmh = malloc(sizeof(struct sgx_vm_handle), 1044 M_SGX, M_WAITOK | M_ZERO); 1045 vmh->sc = sc; 1046 vmh->size = mapsize; 1047 vmh->mem = cdev_pager_allocate(vmh, OBJT_MGTDEVICE, &sgx_pg_ops, 1048 mapsize, nprot, *offset, NULL); 1049 if (vmh->mem == NULL) { 1050 free(vmh, M_SGX); 1051 return (ENOMEM); 1052 } 1053 1054 VM_OBJECT_WLOCK(vmh->mem); 1055 vm_object_set_flag(vmh->mem, OBJ_PG_DTOR); 1056 VM_OBJECT_WUNLOCK(vmh->mem); 1057 1058 *objp = vmh->mem; 1059 1060 return (0); 1061 } 1062 1063 static struct cdevsw sgx_cdevsw = { 1064 .d_version = D_VERSION, 1065 .d_ioctl = sgx_ioctl, 1066 .d_mmap_single = sgx_mmap_single, 1067 .d_name = "Intel SGX", 1068 }; 1069 1070 static int 1071 sgx_get_epc_area(struct sgx_softc *sc) 1072 { 1073 vm_offset_t epc_base_vaddr; 1074 u_int cp[4]; 1075 int error; 1076 int i; 1077 1078 cpuid_count(SGX_CPUID, 0x2, cp); 1079 1080 sc->epc_base = ((uint64_t)(cp[1] & 0xfffff) << 32) + 1081 (cp[0] & 0xfffff000); 1082 sc->epc_size = ((uint64_t)(cp[3] & 0xfffff) << 32) + 1083 (cp[2] & 0xfffff000); 1084 sc->npages = sc->epc_size / SGX_PAGE_SIZE; 1085 1086 if (sc->epc_size == 0 || sc->epc_base == 0) { 1087 printf("%s: Incorrect EPC data: EPC base %lx, size %lu\n", 1088 __func__, sc->epc_base, sc->epc_size); 1089 return (EINVAL); 1090 } 1091 1092 if (cp[3] & 0xffff) 1093 sc->enclave_size_max = (1 << ((cp[3] >> 8) & 0xff)); 1094 else 1095 sc->enclave_size_max = SGX_ENCL_SIZE_MAX_DEF; 1096 1097 epc_base_vaddr = (vm_offset_t)pmap_mapdev_attr(sc->epc_base, 1098 sc->epc_size, VM_MEMATTR_DEFAULT); 1099 1100 sc->epc_pages = malloc(sizeof(struct epc_page) * sc->npages, 1101 M_DEVBUF, M_WAITOK | M_ZERO); 1102 1103 for (i = 0; i < sc->npages; i++) { 1104 sc->epc_pages[i].base = epc_base_vaddr + SGX_PAGE_SIZE * i; 1105 sc->epc_pages[i].phys = sc->epc_base + SGX_PAGE_SIZE * i; 1106 sc->epc_pages[i].index = i; 1107 } 1108 1109 sc->vmem_epc = vmem_create("SGX EPC", sc->epc_base, sc->epc_size, 1110 PAGE_SIZE, PAGE_SIZE, M_FIRSTFIT | M_WAITOK); 1111 if (sc->vmem_epc == NULL) { 1112 printf("%s: Can't create vmem arena.\n", __func__); 1113 free(sc->epc_pages, M_SGX); 1114 return (EINVAL); 1115 } 1116 1117 error = vm_phys_fictitious_reg_range(sc->epc_base, 1118 sc->epc_base + sc->epc_size, VM_MEMATTR_DEFAULT); 1119 if (error) { 1120 printf("%s: Can't register fictitious space.\n", __func__); 1121 free(sc->epc_pages, M_SGX); 1122 return (EINVAL); 1123 } 1124 1125 return (0); 1126 } 1127 1128 static void 1129 sgx_put_epc_area(struct sgx_softc *sc) 1130 { 1131 1132 vm_phys_fictitious_unreg_range(sc->epc_base, 1133 sc->epc_base + sc->epc_size); 1134 1135 free(sc->epc_pages, M_SGX); 1136 } 1137 1138 static int 1139 sgx_load(void) 1140 { 1141 struct sgx_softc *sc; 1142 int error; 1143 1144 sc = &sgx_sc; 1145 1146 if ((cpu_stdext_feature & CPUID_STDEXT_SGX) == 0) 1147 return (ENXIO); 1148 1149 error = sgx_get_epc_area(sc); 1150 if (error) { 1151 printf("%s: Failed to get Processor Reserved Memory area.\n", 1152 __func__); 1153 return (ENXIO); 1154 } 1155 1156 mtx_init(&sc->mtx_encls, "SGX ENCLS", NULL, MTX_DEF); 1157 mtx_init(&sc->mtx, "SGX driver", NULL, MTX_DEF); 1158 1159 TAILQ_INIT(&sc->enclaves); 1160 1161 sc->sgx_cdev = make_dev(&sgx_cdevsw, 0, UID_ROOT, GID_WHEEL, 1162 0600, "isgx"); 1163 1164 sc->state |= SGX_STATE_RUNNING; 1165 1166 printf("SGX initialized: EPC base 0x%lx size %ld (%d pages)\n", 1167 sc->epc_base, sc->epc_size, sc->npages); 1168 1169 return (0); 1170 } 1171 1172 static int 1173 sgx_unload(void) 1174 { 1175 struct sgx_softc *sc; 1176 1177 sc = &sgx_sc; 1178 1179 if ((sc->state & SGX_STATE_RUNNING) == 0) 1180 return (0); 1181 1182 mtx_lock(&sc->mtx); 1183 if (!TAILQ_EMPTY(&sc->enclaves)) { 1184 mtx_unlock(&sc->mtx); 1185 return (EBUSY); 1186 } 1187 sc->state &= ~SGX_STATE_RUNNING; 1188 mtx_unlock(&sc->mtx); 1189 1190 destroy_dev(sc->sgx_cdev); 1191 1192 vmem_destroy(sc->vmem_epc); 1193 sgx_put_epc_area(sc); 1194 1195 mtx_destroy(&sc->mtx_encls); 1196 mtx_destroy(&sc->mtx); 1197 1198 return (0); 1199 } 1200 1201 static int 1202 sgx_handler(module_t mod, int what, void *arg) 1203 { 1204 int error; 1205 1206 switch (what) { 1207 case MOD_LOAD: 1208 error = sgx_load(); 1209 break; 1210 case MOD_UNLOAD: 1211 error = sgx_unload(); 1212 break; 1213 default: 1214 error = 0; 1215 break; 1216 } 1217 1218 return (error); 1219 } 1220 1221 static moduledata_t sgx_kmod = { 1222 "sgx", 1223 sgx_handler, 1224 NULL 1225 }; 1226 1227 DECLARE_MODULE(sgx, sgx_kmod, SI_SUB_LAST, SI_ORDER_ANY); 1228 MODULE_VERSION(sgx, 1); 1229