1 /*-
2 * Copyright (c) 2017 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * This software was developed by BAE Systems, the University of Cambridge
6 * Computer Laboratory, and Memorial University under DARPA/AFRL contract
7 * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing
8 * (TC) research program.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * Design overview.
34 *
35 * The driver provides character device for mmap(2) and ioctl(2) system calls
36 * allowing user to manage isolated compartments ("enclaves") in user VA space.
37 *
38 * The driver duties is EPC pages management, enclave management, user data
39 * validation.
40 *
41 * This driver requires Intel SGX support from hardware.
42 *
43 * /dev/sgx:
44 * .mmap:
45 * sgx_mmap_single() allocates VM object with following pager
46 * operations:
47 * a) sgx_pg_ctor():
48 * VM object constructor does nothing
49 * b) sgx_pg_dtor():
50 * VM object destructor destroys the SGX enclave associated
51 * with the object: it frees all the EPC pages allocated for
52 * enclave and removes the enclave.
53 * c) sgx_pg_fault():
54 * VM object fault handler does nothing
55 *
56 * .ioctl:
57 * sgx_ioctl():
58 * a) SGX_IOC_ENCLAVE_CREATE
59 * Adds Enclave SECS page: initial step of enclave creation.
60 * b) SGX_IOC_ENCLAVE_ADD_PAGE
61 * Adds TCS, REG pages to the enclave.
62 * c) SGX_IOC_ENCLAVE_INIT
63 * Finalizes enclave creation.
64 *
65 * Enclave lifecycle:
66 * .-- ECREATE -- Add SECS page
67 * Kernel | EADD -- Add TCS, REG pages
68 * space | EEXTEND -- Measure the page (take unique hash)
69 * ENCLS | EPA -- Allocate version array page
70 * '-- EINIT -- Finalize enclave creation
71 * User .-- EENTER -- Go to entry point of enclave
72 * space | EEXIT -- Exit back to main application
73 * ENCLU '-- ERESUME -- Resume enclave execution (e.g. after exception)
74 *
75 * Enclave lifecycle from driver point of view:
76 * 1) User calls mmap() on /dev/sgx: we allocate a VM object
77 * 2) User calls ioctl SGX_IOC_ENCLAVE_CREATE: we look for the VM object
78 * associated with user process created on step 1, create SECS physical
79 * page and store it in enclave's VM object queue by special index
80 * SGX_SECS_VM_OBJECT_INDEX.
81 * 3) User calls ioctl SGX_IOC_ENCLAVE_ADD_PAGE: we look for enclave created
82 * on step 2, create TCS or REG physical page and map it to specified by
83 * user address of enclave VM object.
84 * 4) User finalizes enclave creation with ioctl SGX_IOC_ENCLAVE_INIT call.
85 * 5) User can freely enter to and exit from enclave using ENCLU instructions
86 * from userspace: the driver does nothing here.
87 * 6) User proceed munmap(2) system call (or the process with enclave dies):
88 * we destroy the enclave associated with the object.
89 *
90 * EPC page types and their indexes in VM object queue:
91 * - PT_SECS index is special and equals SGX_SECS_VM_OBJECT_INDEX (-1);
92 * - PT_TCS and PT_REG indexes are specified by user in addr field of ioctl
93 * request data and determined as follows:
94 * pidx = OFF_TO_IDX(addp->addr - vmh->base);
95 * - PT_VA index is special, created for PT_REG, PT_TCS and PT_SECS pages
96 * and determined by formula:
97 * va_page_idx = - SGX_VA_PAGES_OFFS - (page_idx / SGX_VA_PAGE_SLOTS);
98 * PT_VA page can hold versions of up to 512 pages, and slot for each
99 * page in PT_VA page is determined as follows:
100 * va_slot_idx = page_idx % SGX_VA_PAGE_SLOTS;
101 * - PT_TRIM is unused.
102 *
103 * Locking:
104 * SGX ENCLS set of instructions have limitations on concurrency:
105 * some instructions can't be executed same time on different CPUs.
106 * We use sc->mtx_encls lock around them to prevent concurrent execution.
107 * sc->mtx lock is used to manage list of created enclaves and the state of
108 * SGX driver.
109 *
110 * Eviction of EPC pages:
111 * Eviction support is not implemented in this driver, however the driver
112 * manages VA (version array) pages: it allocates a VA slot for each EPC
113 * page. This will be required for eviction support in future.
114 * VA pages and slots are currently unused.
115 *
116 * Intel® 64 and IA-32 Architectures Software Developer's Manual
117 * https://software.intel.com/en-us/articles/intel-sdm
118 */
119
120 #include <sys/param.h>
121 #include <sys/systm.h>
122 #include <sys/ioccom.h>
123 #include <sys/malloc.h>
124 #include <sys/kernel.h>
125 #include <sys/lock.h>
126 #include <sys/mutex.h>
127 #include <sys/rwlock.h>
128 #include <sys/conf.h>
129 #include <sys/module.h>
130 #include <sys/proc.h>
131 #include <sys/vmem.h>
132 #include <sys/vmmeter.h>
133
134 #include <vm/vm.h>
135 #include <vm/vm_param.h>
136 #include <vm/vm_extern.h>
137 #include <vm/vm_kern.h>
138 #include <vm/vm_page.h>
139 #include <vm/vm_map.h>
140 #include <vm/vm_object.h>
141 #include <vm/vm_pager.h>
142 #include <vm/vm_phys.h>
143 #include <vm/vm_radix.h>
144 #include <vm/pmap.h>
145
146 #include <machine/md_var.h>
147 #include <machine/specialreg.h>
148 #include <machine/cpufunc.h>
149 #include <machine/sgx.h>
150 #include <machine/sgxreg.h>
151
152 #include <amd64/sgx/sgxvar.h>
153
154 #define SGX_DEBUG
155 #undef SGX_DEBUG
156
157 #ifdef SGX_DEBUG
158 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
159 #else
160 #define dprintf(fmt, ...)
161 #endif
162
163 static struct cdev_pager_ops sgx_pg_ops;
164 struct sgx_softc sgx_sc;
165
166 static int
sgx_get_epc_page(struct sgx_softc * sc,struct epc_page ** epc)167 sgx_get_epc_page(struct sgx_softc *sc, struct epc_page **epc)
168 {
169 vmem_addr_t addr;
170 int i;
171
172 if (vmem_alloc(sc->vmem_epc, PAGE_SIZE, M_FIRSTFIT | M_NOWAIT,
173 &addr) == 0) {
174 i = (addr - sc->epc_base) / PAGE_SIZE;
175 *epc = &sc->epc_pages[i];
176 return (0);
177 }
178
179 return (ENOMEM);
180 }
181
182 static void
sgx_put_epc_page(struct sgx_softc * sc,struct epc_page * epc)183 sgx_put_epc_page(struct sgx_softc *sc, struct epc_page *epc)
184 {
185 vmem_addr_t addr;
186
187 if (epc == NULL)
188 return;
189
190 addr = (epc->index * PAGE_SIZE) + sc->epc_base;
191 vmem_free(sc->vmem_epc, addr, PAGE_SIZE);
192 }
193
194 static void
sgx_insert_epc_page_by_index(vm_page_t page,vm_object_t object,vm_pindex_t pidx,struct pctrie_iter * pages)195 sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
196 vm_pindex_t pidx, struct pctrie_iter *pages)
197 {
198
199 VM_OBJECT_ASSERT_WLOCKED(object);
200
201 page->valid = VM_PAGE_BITS_ALL;
202 vm_page_iter_insert(page, object, pidx, pages);
203 }
204
205 static int
sgx_va_slot_init_by_index(struct sgx_softc * sc,vm_object_t object,uint64_t idx,struct pctrie_iter * pages)206 sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
207 uint64_t idx, struct pctrie_iter *pages)
208 {
209 struct epc_page *epc;
210 vm_page_t page;
211 vm_page_t p;
212 int ret;
213
214 VM_OBJECT_ASSERT_WLOCKED(object);
215
216 p = vm_radix_iter_lookup(pages, idx);
217 if (p == NULL) {
218 ret = sgx_get_epc_page(sc, &epc);
219 if (ret) {
220 dprintf("%s: No free EPC pages available.\n",
221 __func__);
222 return (ret);
223 }
224
225 mtx_lock(&sc->mtx_encls);
226 sgx_epa((void *)epc->base);
227 mtx_unlock(&sc->mtx_encls);
228
229 page = PHYS_TO_VM_PAGE(epc->phys);
230 sgx_insert_epc_page_by_index(page, object, idx, pages);
231 }
232
233 return (0);
234 }
235
236 static int
sgx_va_slot_init(struct sgx_softc * sc,struct sgx_enclave * enclave,vm_pindex_t pidx,struct pctrie_iter * pages)237 sgx_va_slot_init(struct sgx_softc *sc, struct sgx_enclave *enclave,
238 vm_pindex_t pidx, struct pctrie_iter *pages)
239 {
240 uint64_t va_page_idx;
241 uint64_t idx;
242 vm_object_t object;
243 int ret;
244
245 object = enclave->object;
246
247 VM_OBJECT_ASSERT_WLOCKED(object);
248
249 va_page_idx = pidx / SGX_VA_PAGE_SLOTS;
250 idx = - SGX_VA_PAGES_OFFS - va_page_idx;
251
252 ret = sgx_va_slot_init_by_index(sc, object, idx, pages);
253
254 return (ret);
255 }
256
257 static int
sgx_mem_find(struct sgx_softc * sc,uint64_t addr,vm_map_entry_t * entry0,vm_object_t * object0)258 sgx_mem_find(struct sgx_softc *sc, uint64_t addr,
259 vm_map_entry_t *entry0, vm_object_t *object0)
260 {
261 vm_map_t map;
262 vm_map_entry_t entry;
263 vm_object_t object;
264
265 map = &curproc->p_vmspace->vm_map;
266
267 vm_map_lock_read(map);
268 if (!vm_map_lookup_entry(map, addr, &entry)) {
269 vm_map_unlock_read(map);
270 dprintf("%s: Can't find enclave.\n", __func__);
271 return (EINVAL);
272 }
273
274 object = entry->object.vm_object;
275 if (object == NULL || object->handle == NULL) {
276 vm_map_unlock_read(map);
277 return (EINVAL);
278 }
279
280 if (object->type != OBJT_MGTDEVICE ||
281 object->un_pager.devp.ops != &sgx_pg_ops) {
282 vm_map_unlock_read(map);
283 return (EINVAL);
284 }
285
286 vm_object_reference(object);
287
288 *object0 = object;
289 *entry0 = entry;
290 vm_map_unlock_read(map);
291
292 return (0);
293 }
294
295 static int
sgx_enclave_find(struct sgx_softc * sc,uint64_t addr,struct sgx_enclave ** encl)296 sgx_enclave_find(struct sgx_softc *sc, uint64_t addr,
297 struct sgx_enclave **encl)
298 {
299 struct sgx_vm_handle *vmh;
300 struct sgx_enclave *enclave;
301 vm_map_entry_t entry;
302 vm_object_t object;
303 int ret;
304
305 ret = sgx_mem_find(sc, addr, &entry, &object);
306 if (ret)
307 return (ret);
308
309 vmh = object->handle;
310 if (vmh == NULL) {
311 vm_object_deallocate(object);
312 return (EINVAL);
313 }
314
315 enclave = vmh->enclave;
316 if (enclave == NULL || enclave->object == NULL) {
317 vm_object_deallocate(object);
318 return (EINVAL);
319 }
320
321 *encl = enclave;
322
323 return (0);
324 }
325
326 static int
sgx_enclave_alloc(struct sgx_softc * sc,struct secs * secs,struct sgx_enclave ** enclave0)327 sgx_enclave_alloc(struct sgx_softc *sc, struct secs *secs,
328 struct sgx_enclave **enclave0)
329 {
330 struct sgx_enclave *enclave;
331
332 enclave = malloc(sizeof(struct sgx_enclave),
333 M_SGX, M_WAITOK | M_ZERO);
334
335 enclave->base = secs->base;
336 enclave->size = secs->size;
337
338 *enclave0 = enclave;
339
340 return (0);
341 }
342
343 static void
sgx_epc_page_remove(struct sgx_softc * sc,struct epc_page * epc)344 sgx_epc_page_remove(struct sgx_softc *sc,
345 struct epc_page *epc)
346 {
347
348 mtx_lock(&sc->mtx_encls);
349 sgx_eremove((void *)epc->base);
350 mtx_unlock(&sc->mtx_encls);
351 }
352
353 static void
sgx_page_remove(struct sgx_softc * sc,vm_page_t p,struct pctrie_iter * pages)354 sgx_page_remove(struct sgx_softc *sc, vm_page_t p,
355 struct pctrie_iter *pages)
356 {
357 struct epc_page *epc;
358 vm_paddr_t pa;
359 uint64_t offs;
360
361 if (pages != NULL)
362 (void)vm_page_iter_remove(pages, p);
363 else
364 (void) vm_page_remove(p);
365
366 dprintf("%s: p->pidx %ld\n", __func__, p->pindex);
367
368 pa = VM_PAGE_TO_PHYS(p);
369 epc = &sc->epc_pages[0];
370 offs = (pa - epc->phys) / PAGE_SIZE;
371 epc = &sc->epc_pages[offs];
372
373 sgx_epc_page_remove(sc, epc);
374 sgx_put_epc_page(sc, epc);
375 }
376
377 static void
sgx_enclave_remove(struct sgx_softc * sc,struct sgx_enclave * enclave)378 sgx_enclave_remove(struct sgx_softc *sc,
379 struct sgx_enclave *enclave)
380 {
381 struct pctrie_iter pages;
382 vm_object_t object;
383 vm_page_t p, p_secs;
384
385 mtx_lock(&sc->mtx);
386 TAILQ_REMOVE(&sc->enclaves, enclave, next);
387 mtx_unlock(&sc->mtx);
388
389 object = enclave->object;
390
391 vm_page_iter_init(&pages, object);
392 VM_OBJECT_WLOCK(object);
393
394 /*
395 * First remove all the pages except SECS,
396 * then remove SECS page.
397 */
398 restart:
399 VM_RADIX_FOREACH(p, &pages) {
400 if (p->pindex == SGX_SECS_VM_OBJECT_INDEX)
401 continue;
402 if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
403 pctrie_iter_reset(&pages);
404 goto restart;
405 }
406 sgx_page_remove(sc, p, &pages);
407 }
408 p_secs = vm_page_grab(object, SGX_SECS_VM_OBJECT_INDEX,
409 VM_ALLOC_NOCREAT);
410 /* Now remove SECS page */
411 if (p_secs != NULL)
412 sgx_page_remove(sc, p_secs, NULL);
413
414 KASSERT(object->resident_page_count == 0, ("count"));
415
416 VM_OBJECT_WUNLOCK(object);
417 }
418
419 static int
sgx_measure_page(struct sgx_softc * sc,struct epc_page * secs,struct epc_page * epc,uint16_t mrmask)420 sgx_measure_page(struct sgx_softc *sc, struct epc_page *secs,
421 struct epc_page *epc, uint16_t mrmask)
422 {
423 int i, j;
424 int ret;
425
426 mtx_lock(&sc->mtx_encls);
427
428 for (i = 0, j = 1; i < PAGE_SIZE; i += 0x100, j <<= 1) {
429 if (!(j & mrmask))
430 continue;
431
432 ret = sgx_eextend((void *)secs->base,
433 (void *)(epc->base + i));
434 if (ret == SGX_EFAULT) {
435 mtx_unlock(&sc->mtx_encls);
436 return (ret);
437 }
438 }
439
440 mtx_unlock(&sc->mtx_encls);
441
442 return (0);
443 }
444
445 static int
sgx_secs_validate(struct sgx_softc * sc,struct secs * secs)446 sgx_secs_validate(struct sgx_softc *sc, struct secs *secs)
447 {
448 struct secs_attr *attr;
449 int i;
450
451 if (secs->size == 0)
452 return (EINVAL);
453
454 /* BASEADDR must be naturally aligned on an SECS.SIZE boundary. */
455 if (secs->base & (secs->size - 1))
456 return (EINVAL);
457
458 /* SECS.SIZE must be at least 2 pages. */
459 if (secs->size < 2 * PAGE_SIZE)
460 return (EINVAL);
461
462 if ((secs->size & (secs->size - 1)) != 0)
463 return (EINVAL);
464
465 attr = &secs->attributes;
466
467 if (attr->reserved1 != 0 ||
468 attr->reserved2 != 0 ||
469 attr->reserved3 != 0)
470 return (EINVAL);
471
472 for (i = 0; i < SECS_ATTR_RSV4_SIZE; i++)
473 if (attr->reserved4[i])
474 return (EINVAL);
475
476 /*
477 * Intel® Software Guard Extensions Programming Reference
478 * 6.7.2 Relevant Fields in Various Data Structures
479 * 6.7.2.1 SECS.ATTRIBUTES.XFRM
480 * XFRM[1:0] must be set to 0x3.
481 */
482 if ((attr->xfrm & 0x3) != 0x3)
483 return (EINVAL);
484
485 if (!attr->mode64bit)
486 return (EINVAL);
487
488 if (secs->size > sc->enclave_size_max)
489 return (EINVAL);
490
491 for (i = 0; i < SECS_RSV1_SIZE; i++)
492 if (secs->reserved1[i])
493 return (EINVAL);
494
495 for (i = 0; i < SECS_RSV2_SIZE; i++)
496 if (secs->reserved2[i])
497 return (EINVAL);
498
499 for (i = 0; i < SECS_RSV3_SIZE; i++)
500 if (secs->reserved3[i])
501 return (EINVAL);
502
503 for (i = 0; i < SECS_RSV4_SIZE; i++)
504 if (secs->reserved4[i])
505 return (EINVAL);
506
507 return (0);
508 }
509
510 static int
sgx_tcs_validate(struct tcs * tcs)511 sgx_tcs_validate(struct tcs *tcs)
512 {
513 int i;
514
515 if ((tcs->flags) ||
516 (tcs->ossa & (PAGE_SIZE - 1)) ||
517 (tcs->ofsbasgx & (PAGE_SIZE - 1)) ||
518 (tcs->ogsbasgx & (PAGE_SIZE - 1)) ||
519 ((tcs->fslimit & 0xfff) != 0xfff) ||
520 ((tcs->gslimit & 0xfff) != 0xfff))
521 return (EINVAL);
522
523 for (i = 0; i < nitems(tcs->reserved3); i++)
524 if (tcs->reserved3[i])
525 return (EINVAL);
526
527 return (0);
528 }
529
530 static void
sgx_tcs_dump(struct sgx_softc * sc,struct tcs * t)531 sgx_tcs_dump(struct sgx_softc *sc, struct tcs *t)
532 {
533
534 dprintf("t->flags %lx\n", t->flags);
535 dprintf("t->ossa %lx\n", t->ossa);
536 dprintf("t->cssa %x\n", t->cssa);
537 dprintf("t->nssa %x\n", t->nssa);
538 dprintf("t->oentry %lx\n", t->oentry);
539 dprintf("t->ofsbasgx %lx\n", t->ofsbasgx);
540 dprintf("t->ogsbasgx %lx\n", t->ogsbasgx);
541 dprintf("t->fslimit %x\n", t->fslimit);
542 dprintf("t->gslimit %x\n", t->gslimit);
543 }
544
545 static int
sgx_pg_ctor(void * handle,vm_ooffset_t size,vm_prot_t prot,vm_ooffset_t foff,struct ucred * cred,u_short * color)546 sgx_pg_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
547 vm_ooffset_t foff, struct ucred *cred, u_short *color)
548 {
549 struct sgx_vm_handle *vmh;
550
551 vmh = handle;
552 if (vmh == NULL) {
553 dprintf("%s: vmh not found.\n", __func__);
554 return (0);
555 }
556
557 dprintf("%s: vmh->base %lx foff 0x%lx size 0x%lx\n",
558 __func__, vmh->base, foff, size);
559
560 return (0);
561 }
562
563 static void
sgx_pg_dtor(void * handle)564 sgx_pg_dtor(void *handle)
565 {
566 struct sgx_vm_handle *vmh;
567 struct sgx_softc *sc;
568
569 vmh = handle;
570 if (vmh == NULL) {
571 dprintf("%s: vmh not found.\n", __func__);
572 return;
573 }
574
575 sc = vmh->sc;
576 if (sc == NULL) {
577 dprintf("%s: sc is NULL\n", __func__);
578 return;
579 }
580
581 if (vmh->enclave == NULL) {
582 dprintf("%s: Enclave not found.\n", __func__);
583 return;
584 }
585
586 sgx_enclave_remove(sc, vmh->enclave);
587
588 free(vmh->enclave, M_SGX);
589 free(vmh, M_SGX);
590 }
591
592 static int
sgx_pg_fault(vm_object_t object,vm_ooffset_t offset,int prot,vm_page_t * mres)593 sgx_pg_fault(vm_object_t object, vm_ooffset_t offset,
594 int prot, vm_page_t *mres)
595 {
596
597 /*
598 * The purpose of this trivial handler is to handle the race
599 * when user tries to access mmaped region before or during
600 * enclave creation ioctl calls.
601 */
602
603 dprintf("%s: offset 0x%lx\n", __func__, offset);
604
605 return (VM_PAGER_FAIL);
606 }
607
608 static void
sgx_pg_path(void * handle,char * path,size_t len)609 sgx_pg_path(void *handle, char *path, size_t len)
610 {
611 strlcpy(path, "sgx", len);
612 }
613
614 static struct cdev_pager_ops sgx_pg_ops = {
615 .cdev_pg_ctor = sgx_pg_ctor,
616 .cdev_pg_dtor = sgx_pg_dtor,
617 .cdev_pg_fault = sgx_pg_fault,
618 .cdev_pg_path = sgx_pg_path,
619 };
620
621 static void
sgx_insert_epc_page(struct sgx_enclave * enclave,struct epc_page * epc,uint64_t addr,struct pctrie_iter * pages)622 sgx_insert_epc_page(struct sgx_enclave *enclave, struct epc_page *epc,
623 uint64_t addr, struct pctrie_iter *pages)
624 {
625 vm_pindex_t pidx;
626 vm_page_t page;
627
628 VM_OBJECT_ASSERT_WLOCKED(enclave->object);
629
630 pidx = OFF_TO_IDX(addr);
631 page = PHYS_TO_VM_PAGE(epc->phys);
632
633 sgx_insert_epc_page_by_index(page, enclave->object, pidx, pages);
634 }
635
636 static int
sgx_ioctl_create(struct sgx_softc * sc,struct sgx_enclave_create * param)637 sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
638 {
639 struct pctrie_iter pages;
640 struct sgx_vm_handle *vmh;
641 vm_map_entry_t entry;
642 vm_page_t p;
643 struct page_info pginfo;
644 struct secinfo secinfo;
645 struct sgx_enclave *enclave;
646 struct epc_page *epc;
647 struct secs *secs;
648 vm_object_t object;
649 vm_page_t page;
650 int ret;
651
652 epc = NULL;
653 secs = NULL;
654 enclave = NULL;
655 object = NULL;
656
657 /* SGX Enclave Control Structure (SECS) */
658 secs = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO);
659 ret = copyin((void *)param->src, secs, sizeof(struct secs));
660 if (ret) {
661 dprintf("%s: Can't copy SECS.\n", __func__);
662 goto error;
663 }
664
665 ret = sgx_secs_validate(sc, secs);
666 if (ret) {
667 dprintf("%s: SECS validation failed.\n", __func__);
668 goto error;
669 }
670
671 ret = sgx_mem_find(sc, secs->base, &entry, &object);
672 if (ret) {
673 dprintf("%s: Can't find vm_map.\n", __func__);
674 goto error;
675 }
676
677 vmh = object->handle;
678 if (!vmh) {
679 dprintf("%s: Can't find vmh.\n", __func__);
680 ret = ENXIO;
681 goto error;
682 }
683
684 dprintf("%s: entry start %lx offset %lx\n",
685 __func__, entry->start, entry->offset);
686 vmh->base = (entry->start - entry->offset);
687
688 ret = sgx_enclave_alloc(sc, secs, &enclave);
689 if (ret) {
690 dprintf("%s: Can't alloc enclave.\n", __func__);
691 goto error;
692 }
693 enclave->object = object;
694 enclave->vmh = vmh;
695
696 memset(&secinfo, 0, sizeof(struct secinfo));
697 memset(&pginfo, 0, sizeof(struct page_info));
698 pginfo.linaddr = 0;
699 pginfo.srcpge = (uint64_t)secs;
700 pginfo.secinfo = &secinfo;
701 pginfo.secs = 0;
702
703 ret = sgx_get_epc_page(sc, &epc);
704 if (ret) {
705 dprintf("%s: Failed to get free epc page.\n", __func__);
706 goto error;
707 }
708 enclave->secs_epc_page = epc;
709
710 vm_page_iter_init(&pages, object);
711 VM_OBJECT_WLOCK(object);
712 p = vm_radix_iter_lookup(&pages, SGX_SECS_VM_OBJECT_INDEX);
713 if (p) {
714 VM_OBJECT_WUNLOCK(object);
715 /* SECS page already added. */
716 ret = ENXIO;
717 goto error;
718 }
719
720 ret = sgx_va_slot_init_by_index(sc, object,
721 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, &pages);
722 if (ret) {
723 VM_OBJECT_WUNLOCK(object);
724 dprintf("%s: Can't init va slot.\n", __func__);
725 goto error;
726 }
727
728 mtx_lock(&sc->mtx);
729 if ((sc->state & SGX_STATE_RUNNING) == 0) {
730 mtx_unlock(&sc->mtx);
731 /* Remove VA page that was just created for SECS page. */
732 p = vm_page_grab(enclave->object,
733 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX,
734 VM_ALLOC_NOCREAT);
735 sgx_page_remove(sc, p, NULL);
736 VM_OBJECT_WUNLOCK(object);
737 goto error;
738 }
739 mtx_lock(&sc->mtx_encls);
740 ret = sgx_ecreate(&pginfo, (void *)epc->base);
741 mtx_unlock(&sc->mtx_encls);
742 if (ret == SGX_EFAULT) {
743 dprintf("%s: gp fault\n", __func__);
744 mtx_unlock(&sc->mtx);
745 /* Remove VA page that was just created for SECS page. */
746 p = vm_page_grab(enclave->object,
747 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX,
748 VM_ALLOC_NOCREAT);
749 sgx_page_remove(sc, p, NULL);
750 VM_OBJECT_WUNLOCK(object);
751 goto error;
752 }
753
754 TAILQ_INSERT_TAIL(&sc->enclaves, enclave, next);
755 mtx_unlock(&sc->mtx);
756
757 vmh->enclave = enclave;
758
759 page = PHYS_TO_VM_PAGE(epc->phys);
760 sgx_insert_epc_page_by_index(page, enclave->object,
761 SGX_SECS_VM_OBJECT_INDEX, &pages);
762
763 VM_OBJECT_WUNLOCK(object);
764
765 /* Release the reference. */
766 vm_object_deallocate(object);
767
768 free(secs, M_SGX);
769
770 return (0);
771
772 error:
773 free(secs, M_SGX);
774 sgx_put_epc_page(sc, epc);
775 free(enclave, M_SGX);
776 vm_object_deallocate(object);
777
778 return (ret);
779 }
780
781 static int
sgx_ioctl_add_page(struct sgx_softc * sc,struct sgx_enclave_add_page * addp)782 sgx_ioctl_add_page(struct sgx_softc *sc,
783 struct sgx_enclave_add_page *addp)
784 {
785 struct pctrie_iter pages;
786 struct epc_page *secs_epc_page;
787 struct sgx_enclave *enclave;
788 struct sgx_vm_handle *vmh;
789 struct epc_page *epc;
790 struct page_info pginfo;
791 struct secinfo secinfo;
792 vm_object_t object;
793 void *tmp_vaddr;
794 uint64_t page_type;
795 struct tcs *t;
796 uint64_t addr;
797 uint64_t pidx;
798 vm_page_t p;
799 int ret;
800
801 tmp_vaddr = NULL;
802 epc = NULL;
803 object = NULL;
804
805 /* Find and get reference to VM object. */
806 ret = sgx_enclave_find(sc, addp->addr, &enclave);
807 if (ret) {
808 dprintf("%s: Failed to find enclave.\n", __func__);
809 goto error;
810 }
811
812 object = enclave->object;
813 KASSERT(object != NULL, ("vm object is NULL\n"));
814 vmh = object->handle;
815
816 ret = sgx_get_epc_page(sc, &epc);
817 if (ret) {
818 dprintf("%s: Failed to get free epc page.\n", __func__);
819 goto error;
820 }
821
822 memset(&secinfo, 0, sizeof(struct secinfo));
823 ret = copyin((void *)addp->secinfo, &secinfo,
824 sizeof(struct secinfo));
825 if (ret) {
826 dprintf("%s: Failed to copy secinfo.\n", __func__);
827 goto error;
828 }
829
830 tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO);
831 ret = copyin((void *)addp->src, tmp_vaddr, PAGE_SIZE);
832 if (ret) {
833 dprintf("%s: Failed to copy page.\n", __func__);
834 goto error;
835 }
836
837 page_type = (secinfo.flags & SECINFO_FLAGS_PT_M) >>
838 SECINFO_FLAGS_PT_S;
839 if (page_type != SGX_PT_TCS && page_type != SGX_PT_REG) {
840 dprintf("%s: page can't be added.\n", __func__);
841 goto error;
842 }
843 if (page_type == SGX_PT_TCS) {
844 t = (struct tcs *)tmp_vaddr;
845 ret = sgx_tcs_validate(t);
846 if (ret) {
847 dprintf("%s: TCS page validation failed.\n",
848 __func__);
849 goto error;
850 }
851 sgx_tcs_dump(sc, t);
852 }
853
854 addr = (addp->addr - vmh->base);
855 pidx = OFF_TO_IDX(addr);
856
857 vm_page_iter_init(&pages, object);
858 VM_OBJECT_WLOCK(object);
859 p = vm_radix_iter_lookup(&pages, pidx);
860 if (p) {
861 VM_OBJECT_WUNLOCK(object);
862 /* Page already added. */
863 ret = ENXIO;
864 goto error;
865 }
866
867 ret = sgx_va_slot_init(sc, enclave, pidx, &pages);
868 if (ret) {
869 VM_OBJECT_WUNLOCK(object);
870 dprintf("%s: Can't init va slot.\n", __func__);
871 goto error;
872 }
873
874 secs_epc_page = enclave->secs_epc_page;
875 memset(&pginfo, 0, sizeof(struct page_info));
876 pginfo.linaddr = (uint64_t)addp->addr;
877 pginfo.srcpge = (uint64_t)tmp_vaddr;
878 pginfo.secinfo = &secinfo;
879 pginfo.secs = (uint64_t)secs_epc_page->base;
880
881 mtx_lock(&sc->mtx_encls);
882 ret = sgx_eadd(&pginfo, (void *)epc->base);
883 if (ret == SGX_EFAULT) {
884 dprintf("%s: gp fault on eadd\n", __func__);
885 mtx_unlock(&sc->mtx_encls);
886 VM_OBJECT_WUNLOCK(object);
887 goto error;
888 }
889 mtx_unlock(&sc->mtx_encls);
890
891 ret = sgx_measure_page(sc, enclave->secs_epc_page, epc, addp->mrmask);
892 if (ret == SGX_EFAULT) {
893 dprintf("%s: gp fault on eextend\n", __func__);
894 sgx_epc_page_remove(sc, epc);
895 VM_OBJECT_WUNLOCK(object);
896 goto error;
897 }
898
899 sgx_insert_epc_page(enclave, epc, addr, &pages);
900
901 VM_OBJECT_WUNLOCK(object);
902
903 /* Release the reference. */
904 vm_object_deallocate(object);
905
906 free(tmp_vaddr, M_SGX);
907
908 return (0);
909
910 error:
911 free(tmp_vaddr, M_SGX);
912 sgx_put_epc_page(sc, epc);
913 vm_object_deallocate(object);
914
915 return (ret);
916 }
917
918 static int
sgx_ioctl_init(struct sgx_softc * sc,struct sgx_enclave_init * initp)919 sgx_ioctl_init(struct sgx_softc *sc, struct sgx_enclave_init *initp)
920 {
921 struct epc_page *secs_epc_page;
922 struct sgx_enclave *enclave;
923 struct thread *td;
924 void *tmp_vaddr;
925 void *einittoken;
926 void *sigstruct;
927 vm_object_t object;
928 int retry;
929 int ret;
930
931 td = curthread;
932 tmp_vaddr = NULL;
933 object = NULL;
934
935 dprintf("%s: addr %lx, sigstruct %lx, einittoken %lx\n",
936 __func__, initp->addr, initp->sigstruct, initp->einittoken);
937
938 /* Find and get reference to VM object. */
939 ret = sgx_enclave_find(sc, initp->addr, &enclave);
940 if (ret) {
941 dprintf("%s: Failed to find enclave.\n", __func__);
942 goto error;
943 }
944
945 object = enclave->object;
946
947 tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO);
948 sigstruct = tmp_vaddr;
949 einittoken = (void *)((uint64_t)sigstruct + PAGE_SIZE / 2);
950
951 ret = copyin((void *)initp->sigstruct, sigstruct,
952 SGX_SIGSTRUCT_SIZE);
953 if (ret) {
954 dprintf("%s: Failed to copy SIGSTRUCT page.\n", __func__);
955 goto error;
956 }
957
958 ret = copyin((void *)initp->einittoken, einittoken,
959 SGX_EINITTOKEN_SIZE);
960 if (ret) {
961 dprintf("%s: Failed to copy EINITTOKEN page.\n", __func__);
962 goto error;
963 }
964
965 secs_epc_page = enclave->secs_epc_page;
966 retry = 16;
967 do {
968 mtx_lock(&sc->mtx_encls);
969 ret = sgx_einit(sigstruct, (void *)secs_epc_page->base,
970 einittoken);
971 mtx_unlock(&sc->mtx_encls);
972 dprintf("%s: sgx_einit returned %d\n", __func__, ret);
973 } while (ret == SGX_UNMASKED_EVENT && retry--);
974
975 if (ret) {
976 dprintf("%s: Failed init enclave: %d\n", __func__, ret);
977 td->td_retval[0] = ret;
978 ret = 0;
979 }
980
981 error:
982 free(tmp_vaddr, M_SGX);
983
984 /* Release the reference. */
985 vm_object_deallocate(object);
986
987 return (ret);
988 }
989
990 static int
sgx_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flags,struct thread * td)991 sgx_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
992 struct thread *td)
993 {
994 struct sgx_enclave_add_page *addp;
995 struct sgx_enclave_create *param;
996 struct sgx_enclave_init *initp;
997 struct sgx_softc *sc;
998 int ret;
999 int len;
1000
1001 sc = &sgx_sc;
1002
1003 len = IOCPARM_LEN(cmd);
1004
1005 dprintf("%s: cmd %lx, addr %lx, len %d\n",
1006 __func__, cmd, (uint64_t)addr, len);
1007
1008 if (len > SGX_IOCTL_MAX_DATA_LEN)
1009 return (EINVAL);
1010
1011 switch (cmd) {
1012 case SGX_IOC_ENCLAVE_CREATE:
1013 param = (struct sgx_enclave_create *)addr;
1014 ret = sgx_ioctl_create(sc, param);
1015 break;
1016 case SGX_IOC_ENCLAVE_ADD_PAGE:
1017 addp = (struct sgx_enclave_add_page *)addr;
1018 ret = sgx_ioctl_add_page(sc, addp);
1019 break;
1020 case SGX_IOC_ENCLAVE_INIT:
1021 initp = (struct sgx_enclave_init *)addr;
1022 ret = sgx_ioctl_init(sc, initp);
1023 break;
1024 default:
1025 return (EINVAL);
1026 }
1027
1028 return (ret);
1029 }
1030
1031 static int
sgx_mmap_single(struct cdev * cdev,vm_ooffset_t * offset,vm_size_t mapsize,struct vm_object ** objp,int nprot)1032 sgx_mmap_single(struct cdev *cdev, vm_ooffset_t *offset,
1033 vm_size_t mapsize, struct vm_object **objp, int nprot)
1034 {
1035 struct sgx_vm_handle *vmh;
1036 struct sgx_softc *sc;
1037
1038 sc = &sgx_sc;
1039
1040 dprintf("%s: mapsize 0x%lx, offset %lx\n",
1041 __func__, mapsize, *offset);
1042
1043 vmh = malloc(sizeof(struct sgx_vm_handle),
1044 M_SGX, M_WAITOK | M_ZERO);
1045 vmh->sc = sc;
1046 vmh->size = mapsize;
1047 vmh->mem = cdev_pager_allocate(vmh, OBJT_MGTDEVICE, &sgx_pg_ops,
1048 mapsize, nprot, *offset, NULL);
1049 if (vmh->mem == NULL) {
1050 free(vmh, M_SGX);
1051 return (ENOMEM);
1052 }
1053
1054 VM_OBJECT_WLOCK(vmh->mem);
1055 vm_object_set_flag(vmh->mem, OBJ_PG_DTOR);
1056 VM_OBJECT_WUNLOCK(vmh->mem);
1057
1058 *objp = vmh->mem;
1059
1060 return (0);
1061 }
1062
1063 static struct cdevsw sgx_cdevsw = {
1064 .d_version = D_VERSION,
1065 .d_ioctl = sgx_ioctl,
1066 .d_mmap_single = sgx_mmap_single,
1067 .d_name = "Intel SGX",
1068 };
1069
1070 static int
sgx_get_epc_area(struct sgx_softc * sc)1071 sgx_get_epc_area(struct sgx_softc *sc)
1072 {
1073 vm_offset_t epc_base_vaddr;
1074 u_int cp[4];
1075 int error;
1076 int i;
1077
1078 cpuid_count(SGX_CPUID, 0x2, cp);
1079
1080 sc->epc_base = ((uint64_t)(cp[1] & 0xfffff) << 32) +
1081 (cp[0] & 0xfffff000);
1082 sc->epc_size = ((uint64_t)(cp[3] & 0xfffff) << 32) +
1083 (cp[2] & 0xfffff000);
1084 sc->npages = sc->epc_size / SGX_PAGE_SIZE;
1085
1086 if (sc->epc_size == 0 || sc->epc_base == 0) {
1087 printf("%s: Incorrect EPC data: EPC base %lx, size %lu\n",
1088 __func__, sc->epc_base, sc->epc_size);
1089 return (EINVAL);
1090 }
1091
1092 if (cp[3] & 0xffff)
1093 sc->enclave_size_max = (1 << ((cp[3] >> 8) & 0xff));
1094 else
1095 sc->enclave_size_max = SGX_ENCL_SIZE_MAX_DEF;
1096
1097 epc_base_vaddr = (vm_offset_t)pmap_mapdev_attr(sc->epc_base,
1098 sc->epc_size, VM_MEMATTR_DEFAULT);
1099
1100 sc->epc_pages = malloc(sizeof(struct epc_page) * sc->npages,
1101 M_DEVBUF, M_WAITOK | M_ZERO);
1102
1103 for (i = 0; i < sc->npages; i++) {
1104 sc->epc_pages[i].base = epc_base_vaddr + SGX_PAGE_SIZE * i;
1105 sc->epc_pages[i].phys = sc->epc_base + SGX_PAGE_SIZE * i;
1106 sc->epc_pages[i].index = i;
1107 }
1108
1109 sc->vmem_epc = vmem_create("SGX EPC", sc->epc_base, sc->epc_size,
1110 PAGE_SIZE, PAGE_SIZE, M_FIRSTFIT | M_WAITOK);
1111 if (sc->vmem_epc == NULL) {
1112 printf("%s: Can't create vmem arena.\n", __func__);
1113 free(sc->epc_pages, M_SGX);
1114 return (EINVAL);
1115 }
1116
1117 error = vm_phys_fictitious_reg_range(sc->epc_base,
1118 sc->epc_base + sc->epc_size, VM_MEMATTR_DEFAULT);
1119 if (error) {
1120 printf("%s: Can't register fictitious space.\n", __func__);
1121 free(sc->epc_pages, M_SGX);
1122 return (EINVAL);
1123 }
1124
1125 return (0);
1126 }
1127
1128 static void
sgx_put_epc_area(struct sgx_softc * sc)1129 sgx_put_epc_area(struct sgx_softc *sc)
1130 {
1131
1132 vm_phys_fictitious_unreg_range(sc->epc_base,
1133 sc->epc_base + sc->epc_size);
1134
1135 free(sc->epc_pages, M_SGX);
1136 }
1137
1138 static int
sgx_load(void)1139 sgx_load(void)
1140 {
1141 struct sgx_softc *sc;
1142 int error;
1143
1144 sc = &sgx_sc;
1145
1146 if ((cpu_stdext_feature & CPUID_STDEXT_SGX) == 0)
1147 return (ENXIO);
1148
1149 error = sgx_get_epc_area(sc);
1150 if (error) {
1151 printf("%s: Failed to get Processor Reserved Memory area.\n",
1152 __func__);
1153 return (ENXIO);
1154 }
1155
1156 mtx_init(&sc->mtx_encls, "SGX ENCLS", NULL, MTX_DEF);
1157 mtx_init(&sc->mtx, "SGX driver", NULL, MTX_DEF);
1158
1159 TAILQ_INIT(&sc->enclaves);
1160
1161 sc->sgx_cdev = make_dev(&sgx_cdevsw, 0, UID_ROOT, GID_WHEEL,
1162 0600, "isgx");
1163
1164 sc->state |= SGX_STATE_RUNNING;
1165
1166 printf("SGX initialized: EPC base 0x%lx size %ld (%d pages)\n",
1167 sc->epc_base, sc->epc_size, sc->npages);
1168
1169 return (0);
1170 }
1171
1172 static int
sgx_unload(void)1173 sgx_unload(void)
1174 {
1175 struct sgx_softc *sc;
1176
1177 sc = &sgx_sc;
1178
1179 if ((sc->state & SGX_STATE_RUNNING) == 0)
1180 return (0);
1181
1182 mtx_lock(&sc->mtx);
1183 if (!TAILQ_EMPTY(&sc->enclaves)) {
1184 mtx_unlock(&sc->mtx);
1185 return (EBUSY);
1186 }
1187 sc->state &= ~SGX_STATE_RUNNING;
1188 mtx_unlock(&sc->mtx);
1189
1190 destroy_dev(sc->sgx_cdev);
1191
1192 vmem_destroy(sc->vmem_epc);
1193 sgx_put_epc_area(sc);
1194
1195 mtx_destroy(&sc->mtx_encls);
1196 mtx_destroy(&sc->mtx);
1197
1198 return (0);
1199 }
1200
1201 static int
sgx_handler(module_t mod,int what,void * arg)1202 sgx_handler(module_t mod, int what, void *arg)
1203 {
1204 int error;
1205
1206 switch (what) {
1207 case MOD_LOAD:
1208 error = sgx_load();
1209 break;
1210 case MOD_UNLOAD:
1211 error = sgx_unload();
1212 break;
1213 default:
1214 error = 0;
1215 break;
1216 }
1217
1218 return (error);
1219 }
1220
1221 static moduledata_t sgx_kmod = {
1222 "sgx",
1223 sgx_handler,
1224 NULL
1225 };
1226
1227 DECLARE_MODULE(sgx, sgx_kmod, SI_SUB_LAST, SI_ORDER_ANY);
1228 MODULE_VERSION(sgx, 1);
1229