1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2016-20 Intel Corporation. */
3
4 #include <asm/mman.h>
5 #include <asm/sgx.h>
6 #include <crypto/sha2.h>
7 #include <linux/mman.h>
8 #include <linux/delay.h>
9 #include <linux/file.h>
10 #include <linux/hashtable.h>
11 #include <linux/highmem.h>
12 #include <linux/ratelimit.h>
13 #include <linux/sched/signal.h>
14 #include <linux/shmem_fs.h>
15 #include <linux/slab.h>
16 #include <linux/suspend.h>
17 #include "driver.h"
18 #include "encl.h"
19 #include "encls.h"
20
sgx_encl_grow(struct sgx_encl * encl,bool reclaim)21 struct sgx_va_page *sgx_encl_grow(struct sgx_encl *encl, bool reclaim)
22 {
23 struct sgx_va_page *va_page = NULL;
24 void *err;
25
26 BUILD_BUG_ON(SGX_VA_SLOT_COUNT !=
27 (SGX_ENCL_PAGE_VA_OFFSET_MASK >> 3) + 1);
28
29 if (!(encl->page_cnt % SGX_VA_SLOT_COUNT)) {
30 va_page = kzalloc(sizeof(*va_page), GFP_KERNEL);
31 if (!va_page)
32 return ERR_PTR(-ENOMEM);
33
34 va_page->epc_page = sgx_alloc_va_page(reclaim);
35 if (IS_ERR(va_page->epc_page)) {
36 err = ERR_CAST(va_page->epc_page);
37 kfree(va_page);
38 return err;
39 }
40
41 WARN_ON_ONCE(encl->page_cnt % SGX_VA_SLOT_COUNT);
42 }
43 encl->page_cnt++;
44 return va_page;
45 }
46
sgx_encl_shrink(struct sgx_encl * encl,struct sgx_va_page * va_page)47 void sgx_encl_shrink(struct sgx_encl *encl, struct sgx_va_page *va_page)
48 {
49 encl->page_cnt--;
50
51 if (va_page) {
52 sgx_encl_free_epc_page(va_page->epc_page);
53 list_del(&va_page->list);
54 kfree(va_page);
55 }
56 }
57
sgx_encl_create(struct sgx_encl * encl,struct sgx_secs * secs)58 static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs)
59 {
60 struct sgx_epc_page *secs_epc;
61 struct sgx_va_page *va_page;
62 struct sgx_pageinfo pginfo;
63 struct sgx_secinfo secinfo;
64 unsigned long encl_size;
65 struct file *backing;
66 long ret;
67
68 /*
69 * ECREATE would detect this too, but checking here also ensures
70 * that the 'encl_size' calculations below can never overflow.
71 */
72 if (!is_power_of_2(secs->size))
73 return -EINVAL;
74
75 va_page = sgx_encl_grow(encl, true);
76 if (IS_ERR(va_page))
77 return PTR_ERR(va_page);
78 else if (va_page)
79 list_add(&va_page->list, &encl->va_pages);
80 /* else the tail page of the VA page list had free slots. */
81
82 /* The extra page goes to SECS. */
83 encl_size = secs->size + PAGE_SIZE;
84
85 backing = shmem_file_setup("SGX backing", encl_size + (encl_size >> 5),
86 VM_NORESERVE);
87 if (IS_ERR(backing)) {
88 ret = PTR_ERR(backing);
89 goto err_out_shrink;
90 }
91
92 encl->backing = backing;
93
94 secs_epc = sgx_alloc_epc_page(&encl->secs, true);
95 if (IS_ERR(secs_epc)) {
96 ret = PTR_ERR(secs_epc);
97 goto err_out_backing;
98 }
99
100 encl->secs.epc_page = secs_epc;
101
102 pginfo.addr = 0;
103 pginfo.contents = (unsigned long)secs;
104 pginfo.metadata = (unsigned long)&secinfo;
105 pginfo.secs = 0;
106 memset(&secinfo, 0, sizeof(secinfo));
107
108 ret = __ecreate((void *)&pginfo, sgx_get_epc_virt_addr(secs_epc));
109 if (ret) {
110 ret = -EIO;
111 goto err_out;
112 }
113
114 if (secs->attributes & SGX_ATTR_DEBUG)
115 set_bit(SGX_ENCL_DEBUG, &encl->flags);
116
117 encl->secs.encl = encl;
118 encl->secs.type = SGX_PAGE_TYPE_SECS;
119 encl->base = secs->base;
120 encl->size = secs->size;
121 encl->attributes = secs->attributes;
122 encl->attributes_mask = SGX_ATTR_UNPRIV_MASK;
123
124 /* Set only after completion, as encl->lock has not been taken. */
125 set_bit(SGX_ENCL_CREATED, &encl->flags);
126
127 return 0;
128
129 err_out:
130 sgx_encl_free_epc_page(encl->secs.epc_page);
131 encl->secs.epc_page = NULL;
132
133 err_out_backing:
134 fput(encl->backing);
135 encl->backing = NULL;
136
137 err_out_shrink:
138 sgx_encl_shrink(encl, va_page);
139
140 return ret;
141 }
142
143 /**
144 * sgx_ioc_enclave_create() - handler for %SGX_IOC_ENCLAVE_CREATE
145 * @encl: An enclave pointer.
146 * @arg: The ioctl argument.
147 *
148 * Allocate kernel data structures for the enclave and invoke ECREATE.
149 *
150 * Return:
151 * - 0: Success.
152 * - -EIO: ECREATE failed.
153 * - -errno: POSIX error.
154 */
sgx_ioc_enclave_create(struct sgx_encl * encl,void __user * arg)155 static long sgx_ioc_enclave_create(struct sgx_encl *encl, void __user *arg)
156 {
157 struct sgx_enclave_create create_arg;
158 void *secs;
159 int ret;
160
161 if (test_bit(SGX_ENCL_CREATED, &encl->flags))
162 return -EINVAL;
163
164 if (copy_from_user(&create_arg, arg, sizeof(create_arg)))
165 return -EFAULT;
166
167 secs = kmalloc(PAGE_SIZE, GFP_KERNEL);
168 if (!secs)
169 return -ENOMEM;
170
171 if (copy_from_user(secs, (void __user *)create_arg.src, PAGE_SIZE))
172 ret = -EFAULT;
173 else
174 ret = sgx_encl_create(encl, secs);
175
176 kfree(secs);
177 return ret;
178 }
179
sgx_validate_secinfo(struct sgx_secinfo * secinfo)180 static int sgx_validate_secinfo(struct sgx_secinfo *secinfo)
181 {
182 u64 perm = secinfo->flags & SGX_SECINFO_PERMISSION_MASK;
183 u64 pt = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK;
184
185 if (pt != SGX_SECINFO_REG && pt != SGX_SECINFO_TCS)
186 return -EINVAL;
187
188 if ((perm & SGX_SECINFO_W) && !(perm & SGX_SECINFO_R))
189 return -EINVAL;
190
191 /*
192 * CPU will silently overwrite the permissions as zero, which means
193 * that we need to validate it ourselves.
194 */
195 if (pt == SGX_SECINFO_TCS && perm)
196 return -EINVAL;
197
198 if (secinfo->flags & SGX_SECINFO_RESERVED_MASK)
199 return -EINVAL;
200
201 if (memchr_inv(secinfo->reserved, 0, sizeof(secinfo->reserved)))
202 return -EINVAL;
203
204 return 0;
205 }
206
__sgx_encl_add_page(struct sgx_encl * encl,struct sgx_encl_page * encl_page,struct sgx_epc_page * epc_page,struct sgx_secinfo * secinfo,unsigned long src)207 static int __sgx_encl_add_page(struct sgx_encl *encl,
208 struct sgx_encl_page *encl_page,
209 struct sgx_epc_page *epc_page,
210 struct sgx_secinfo *secinfo, unsigned long src)
211 {
212 struct sgx_pageinfo pginfo;
213 struct vm_area_struct *vma;
214 struct page *src_page;
215 int ret;
216
217 /* Deny noexec. */
218 vma = find_vma(current->mm, src);
219 if (!vma)
220 return -EFAULT;
221
222 if (!(vma->vm_flags & VM_MAYEXEC))
223 return -EACCES;
224
225 ret = get_user_pages(src, 1, 0, &src_page);
226 if (ret < 1)
227 return -EFAULT;
228
229 pginfo.secs = (unsigned long)sgx_get_epc_virt_addr(encl->secs.epc_page);
230 pginfo.addr = encl_page->desc & PAGE_MASK;
231 pginfo.metadata = (unsigned long)secinfo;
232 pginfo.contents = (unsigned long)kmap_local_page(src_page);
233
234 ret = __eadd(&pginfo, sgx_get_epc_virt_addr(epc_page));
235
236 kunmap_local((void *)pginfo.contents);
237 put_page(src_page);
238
239 return ret ? -EIO : 0;
240 }
241
242 /*
243 * If the caller requires measurement of the page as a proof for the content,
244 * use EEXTEND to add a measurement for 256 bytes of the page. Repeat this
245 * operation until the entire page is measured."
246 */
__sgx_encl_extend(struct sgx_encl * encl,struct sgx_epc_page * epc_page)247 static int __sgx_encl_extend(struct sgx_encl *encl,
248 struct sgx_epc_page *epc_page)
249 {
250 unsigned long offset;
251 int ret;
252
253 for (offset = 0; offset < PAGE_SIZE; offset += SGX_EEXTEND_BLOCK_SIZE) {
254 ret = __eextend(sgx_get_epc_virt_addr(encl->secs.epc_page),
255 sgx_get_epc_virt_addr(epc_page) + offset);
256 if (ret) {
257 if (encls_failed(ret))
258 ENCLS_WARN(ret, "EEXTEND");
259
260 return -EIO;
261 }
262 }
263
264 return 0;
265 }
266
sgx_encl_add_page(struct sgx_encl * encl,unsigned long src,unsigned long offset,struct sgx_secinfo * secinfo,unsigned long flags)267 static int sgx_encl_add_page(struct sgx_encl *encl, unsigned long src,
268 unsigned long offset, struct sgx_secinfo *secinfo,
269 unsigned long flags)
270 {
271 struct sgx_encl_page *encl_page;
272 struct sgx_epc_page *epc_page;
273 struct sgx_va_page *va_page;
274 int ret;
275
276 encl_page = sgx_encl_page_alloc(encl, offset, secinfo->flags);
277 if (IS_ERR(encl_page))
278 return PTR_ERR(encl_page);
279
280 epc_page = sgx_alloc_epc_page(encl_page, true);
281 if (IS_ERR(epc_page)) {
282 kfree(encl_page);
283 return PTR_ERR(epc_page);
284 }
285
286 va_page = sgx_encl_grow(encl, true);
287 if (IS_ERR(va_page)) {
288 ret = PTR_ERR(va_page);
289 goto err_out_free;
290 }
291
292 mmap_read_lock(current->mm);
293 mutex_lock(&encl->lock);
294
295 /*
296 * Adding to encl->va_pages must be done under encl->lock. Ditto for
297 * deleting (via sgx_encl_shrink()) in the error path.
298 */
299 if (va_page)
300 list_add(&va_page->list, &encl->va_pages);
301
302 /*
303 * Insert prior to EADD in case of OOM. EADD modifies MRENCLAVE, i.e.
304 * can't be gracefully unwound, while failure on EADD/EXTEND is limited
305 * to userspace errors (or kernel/hardware bugs).
306 */
307 ret = xa_insert(&encl->page_array, PFN_DOWN(encl_page->desc),
308 encl_page, GFP_KERNEL);
309 if (ret)
310 goto err_out_unlock;
311
312 ret = __sgx_encl_add_page(encl, encl_page, epc_page, secinfo,
313 src);
314 if (ret)
315 goto err_out;
316
317 /*
318 * Complete the "add" before doing the "extend" so that the "add"
319 * isn't in a half-baked state in the extremely unlikely scenario
320 * the enclave will be destroyed in response to EEXTEND failure.
321 */
322 encl_page->encl = encl;
323 encl_page->epc_page = epc_page;
324 encl_page->type = (secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK) >> 8;
325 encl->secs_child_cnt++;
326
327 if (flags & SGX_PAGE_MEASURE) {
328 ret = __sgx_encl_extend(encl, epc_page);
329 if (ret)
330 goto err_out;
331 }
332
333 sgx_mark_page_reclaimable(encl_page->epc_page);
334 mutex_unlock(&encl->lock);
335 mmap_read_unlock(current->mm);
336 return ret;
337
338 err_out:
339 xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc));
340
341 err_out_unlock:
342 sgx_encl_shrink(encl, va_page);
343 mutex_unlock(&encl->lock);
344 mmap_read_unlock(current->mm);
345
346 err_out_free:
347 sgx_encl_free_epc_page(epc_page);
348 kfree(encl_page);
349
350 return ret;
351 }
352
353 /*
354 * Ensure user provided offset and length values are valid for
355 * an enclave.
356 */
sgx_validate_offset_length(struct sgx_encl * encl,unsigned long offset,unsigned long length)357 static int sgx_validate_offset_length(struct sgx_encl *encl,
358 unsigned long offset,
359 unsigned long length)
360 {
361 if (!IS_ALIGNED(offset, PAGE_SIZE))
362 return -EINVAL;
363
364 if (!length || !IS_ALIGNED(length, PAGE_SIZE))
365 return -EINVAL;
366
367 if (offset + length < offset)
368 return -EINVAL;
369
370 if (offset + length - PAGE_SIZE >= encl->size)
371 return -EINVAL;
372
373 return 0;
374 }
375
376 /**
377 * sgx_ioc_enclave_add_pages() - The handler for %SGX_IOC_ENCLAVE_ADD_PAGES
378 * @encl: an enclave pointer
379 * @arg: a user pointer to a struct sgx_enclave_add_pages instance
380 *
381 * Add one or more pages to an uninitialized enclave, and optionally extend the
382 * measurement with the contents of the page. The SECINFO and measurement mask
383 * are applied to all pages.
384 *
385 * A SECINFO for a TCS is required to always contain zero permissions because
386 * CPU silently zeros them. Allowing anything else would cause a mismatch in
387 * the measurement.
388 *
389 * mmap()'s protection bits are capped by the page permissions. For each page
390 * address, the maximum protection bits are computed with the following
391 * heuristics:
392 *
393 * 1. A regular page: PROT_R, PROT_W and PROT_X match the SECINFO permissions.
394 * 2. A TCS page: PROT_R | PROT_W.
395 *
396 * mmap() is not allowed to surpass the minimum of the maximum protection bits
397 * within the given address range.
398 *
399 * The function deinitializes kernel data structures for enclave and returns
400 * -EIO in any of the following conditions:
401 *
402 * - Enclave Page Cache (EPC), the physical memory holding enclaves, has
403 * been invalidated. This will cause EADD and EEXTEND to fail.
404 * - If the source address is corrupted somehow when executing EADD.
405 *
406 * Return:
407 * - 0: Success.
408 * - -EACCES: The source page is located in a noexec partition.
409 * - -ENOMEM: Out of EPC pages.
410 * - -EINTR: The call was interrupted before data was processed.
411 * - -EIO: Either EADD or EEXTEND failed because invalid source address
412 * or power cycle.
413 * - -errno: POSIX error.
414 */
sgx_ioc_enclave_add_pages(struct sgx_encl * encl,void __user * arg)415 static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg)
416 {
417 struct sgx_enclave_add_pages add_arg;
418 struct sgx_secinfo secinfo;
419 unsigned long c;
420 int ret;
421
422 if (!test_bit(SGX_ENCL_CREATED, &encl->flags) ||
423 test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
424 return -EINVAL;
425
426 if (copy_from_user(&add_arg, arg, sizeof(add_arg)))
427 return -EFAULT;
428
429 if (!IS_ALIGNED(add_arg.src, PAGE_SIZE))
430 return -EINVAL;
431
432 if (sgx_validate_offset_length(encl, add_arg.offset, add_arg.length))
433 return -EINVAL;
434
435 if (copy_from_user(&secinfo, (void __user *)add_arg.secinfo,
436 sizeof(secinfo)))
437 return -EFAULT;
438
439 if (sgx_validate_secinfo(&secinfo))
440 return -EINVAL;
441
442 for (c = 0 ; c < add_arg.length; c += PAGE_SIZE) {
443 if (signal_pending(current)) {
444 if (!c)
445 ret = -ERESTARTSYS;
446
447 break;
448 }
449
450 if (need_resched())
451 cond_resched();
452
453 ret = sgx_encl_add_page(encl, add_arg.src + c, add_arg.offset + c,
454 &secinfo, add_arg.flags);
455 if (ret)
456 break;
457 }
458
459 add_arg.count = c;
460
461 if (copy_to_user(arg, &add_arg, sizeof(add_arg)))
462 return -EFAULT;
463
464 return ret;
465 }
466
sgx_encl_init(struct sgx_encl * encl,struct sgx_sigstruct * sigstruct,void * token)467 static int sgx_encl_init(struct sgx_encl *encl, struct sgx_sigstruct *sigstruct,
468 void *token)
469 {
470 u64 mrsigner[4];
471 int i, j;
472 void *addr;
473 int ret;
474
475 /*
476 * Deny initializing enclaves with attributes (namely provisioning)
477 * that have not been explicitly allowed.
478 */
479 if (encl->attributes & ~encl->attributes_mask)
480 return -EACCES;
481
482 /*
483 * Attributes should not be enforced *only* against what's available on
484 * platform (done in sgx_encl_create) but checked and enforced against
485 * the mask for enforcement in sigstruct. For example an enclave could
486 * opt to sign with AVX bit in xfrm, but still be loadable on a platform
487 * without it if the sigstruct->body.attributes_mask does not turn that
488 * bit on.
489 */
490 if (sigstruct->body.attributes & sigstruct->body.attributes_mask &
491 sgx_attributes_reserved_mask)
492 return -EINVAL;
493
494 if (sigstruct->body.miscselect & sigstruct->body.misc_mask &
495 sgx_misc_reserved_mask)
496 return -EINVAL;
497
498 if (sigstruct->body.xfrm & sigstruct->body.xfrm_mask &
499 sgx_xfrm_reserved_mask)
500 return -EINVAL;
501
502 sha256(sigstruct->modulus, SGX_MODULUS_SIZE, (u8 *)mrsigner);
503
504 mutex_lock(&encl->lock);
505
506 /*
507 * ENCLS[EINIT] is interruptible because it has such a high latency,
508 * e.g. 50k+ cycles on success. If an IRQ/NMI/SMI becomes pending,
509 * EINIT may fail with SGX_UNMASKED_EVENT so that the event can be
510 * serviced.
511 */
512 for (i = 0; i < SGX_EINIT_SLEEP_COUNT; i++) {
513 for (j = 0; j < SGX_EINIT_SPIN_COUNT; j++) {
514 addr = sgx_get_epc_virt_addr(encl->secs.epc_page);
515
516 preempt_disable();
517
518 sgx_update_lepubkeyhash(mrsigner);
519
520 ret = __einit(sigstruct, token, addr);
521
522 preempt_enable();
523
524 if (ret == SGX_UNMASKED_EVENT)
525 continue;
526 else
527 break;
528 }
529
530 if (ret != SGX_UNMASKED_EVENT)
531 break;
532
533 msleep_interruptible(SGX_EINIT_SLEEP_TIME);
534
535 if (signal_pending(current)) {
536 ret = -ERESTARTSYS;
537 goto err_out;
538 }
539 }
540
541 if (encls_faulted(ret)) {
542 if (encls_failed(ret))
543 ENCLS_WARN(ret, "EINIT");
544
545 ret = -EIO;
546 } else if (ret) {
547 pr_debug("EINIT returned %d\n", ret);
548 ret = -EPERM;
549 } else {
550 set_bit(SGX_ENCL_INITIALIZED, &encl->flags);
551 }
552
553 err_out:
554 mutex_unlock(&encl->lock);
555 return ret;
556 }
557
558 /**
559 * sgx_ioc_enclave_init() - handler for %SGX_IOC_ENCLAVE_INIT
560 * @encl: an enclave pointer
561 * @arg: userspace pointer to a struct sgx_enclave_init instance
562 *
563 * Flush any outstanding enqueued EADD operations and perform EINIT. The
564 * Launch Enclave Public Key Hash MSRs are rewritten as necessary to match
565 * the enclave's MRSIGNER, which is calculated from the provided sigstruct.
566 *
567 * Return:
568 * - 0: Success.
569 * - -EPERM: Invalid SIGSTRUCT.
570 * - -EIO: EINIT failed because of a power cycle.
571 * - -errno: POSIX error.
572 */
sgx_ioc_enclave_init(struct sgx_encl * encl,void __user * arg)573 static long sgx_ioc_enclave_init(struct sgx_encl *encl, void __user *arg)
574 {
575 struct sgx_sigstruct *sigstruct;
576 struct sgx_enclave_init init_arg;
577 void *token;
578 int ret;
579
580 if (!test_bit(SGX_ENCL_CREATED, &encl->flags) ||
581 test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
582 return -EINVAL;
583
584 if (copy_from_user(&init_arg, arg, sizeof(init_arg)))
585 return -EFAULT;
586
587 /*
588 * 'sigstruct' must be on a page boundary and 'token' on a 512 byte
589 * boundary. kmalloc() will give this alignment when allocating
590 * PAGE_SIZE bytes.
591 */
592 sigstruct = kmalloc(PAGE_SIZE, GFP_KERNEL);
593 if (!sigstruct)
594 return -ENOMEM;
595
596 token = (void *)((unsigned long)sigstruct + PAGE_SIZE / 2);
597 memset(token, 0, SGX_LAUNCH_TOKEN_SIZE);
598
599 if (copy_from_user(sigstruct, (void __user *)init_arg.sigstruct,
600 sizeof(*sigstruct))) {
601 ret = -EFAULT;
602 goto out;
603 }
604
605 /*
606 * A legacy field used with Intel signed enclaves. These used to mean
607 * regular and architectural enclaves. The CPU only accepts these values
608 * but they do not have any other meaning.
609 *
610 * Thus, reject any other values.
611 */
612 if (sigstruct->header.vendor != 0x0000 &&
613 sigstruct->header.vendor != 0x8086) {
614 ret = -EINVAL;
615 goto out;
616 }
617
618 ret = sgx_encl_init(encl, sigstruct, token);
619
620 out:
621 kfree(sigstruct);
622 return ret;
623 }
624
625 /**
626 * sgx_ioc_enclave_provision() - handler for %SGX_IOC_ENCLAVE_PROVISION
627 * @encl: an enclave pointer
628 * @arg: userspace pointer to a struct sgx_enclave_provision instance
629 *
630 * Allow ATTRIBUTE.PROVISION_KEY for an enclave by providing a file handle to
631 * /dev/sgx_provision.
632 *
633 * Return:
634 * - 0: Success.
635 * - -errno: Otherwise.
636 */
sgx_ioc_enclave_provision(struct sgx_encl * encl,void __user * arg)637 static long sgx_ioc_enclave_provision(struct sgx_encl *encl, void __user *arg)
638 {
639 struct sgx_enclave_provision params;
640
641 if (copy_from_user(¶ms, arg, sizeof(params)))
642 return -EFAULT;
643
644 return sgx_set_attribute(&encl->attributes_mask, params.fd);
645 }
646
647 /*
648 * Ensure enclave is ready for SGX2 functions. Readiness is checked
649 * by ensuring the hardware supports SGX2 and the enclave is initialized
650 * and thus able to handle requests to modify pages within it.
651 */
sgx_ioc_sgx2_ready(struct sgx_encl * encl)652 static int sgx_ioc_sgx2_ready(struct sgx_encl *encl)
653 {
654 if (!(cpu_feature_enabled(X86_FEATURE_SGX2)))
655 return -ENODEV;
656
657 if (!test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
658 return -EINVAL;
659
660 return 0;
661 }
662
663 /*
664 * Some SGX functions require that no cached linear-to-physical address
665 * mappings are present before they can succeed. Collaborate with
666 * hardware via ENCLS[ETRACK] to ensure that all cached
667 * linear-to-physical address mappings belonging to all threads of
668 * the enclave are cleared. See sgx_encl_cpumask() for details.
669 *
670 * Must be called with enclave's mutex held from the time the
671 * SGX function requiring that no cached linear-to-physical mappings
672 * are present is executed until this ETRACK flow is complete.
673 */
sgx_enclave_etrack(struct sgx_encl * encl)674 static int sgx_enclave_etrack(struct sgx_encl *encl)
675 {
676 void *epc_virt;
677 int ret;
678
679 epc_virt = sgx_get_epc_virt_addr(encl->secs.epc_page);
680 ret = __etrack(epc_virt);
681 if (ret) {
682 /*
683 * ETRACK only fails when there is an OS issue. For
684 * example, two consecutive ETRACK was sent without
685 * completed IPI between.
686 */
687 pr_err_once("ETRACK returned %d (0x%x)", ret, ret);
688 /*
689 * Send IPIs to kick CPUs out of the enclave and
690 * try ETRACK again.
691 */
692 on_each_cpu_mask(sgx_encl_cpumask(encl), sgx_ipi_cb, NULL, 1);
693 ret = __etrack(epc_virt);
694 if (ret) {
695 pr_err_once("ETRACK repeat returned %d (0x%x)",
696 ret, ret);
697 return -EFAULT;
698 }
699 }
700 on_each_cpu_mask(sgx_encl_cpumask(encl), sgx_ipi_cb, NULL, 1);
701
702 return 0;
703 }
704
705 /**
706 * sgx_enclave_restrict_permissions() - Restrict EPCM permissions
707 * @encl: Enclave to which the pages belong.
708 * @modp: Checked parameters from user on which pages need modifying and
709 * their new permissions.
710 *
711 * Return:
712 * - 0: Success.
713 * - -errno: Otherwise.
714 */
715 static long
sgx_enclave_restrict_permissions(struct sgx_encl * encl,struct sgx_enclave_restrict_permissions * modp)716 sgx_enclave_restrict_permissions(struct sgx_encl *encl,
717 struct sgx_enclave_restrict_permissions *modp)
718 {
719 struct sgx_encl_page *entry;
720 struct sgx_secinfo secinfo;
721 unsigned long addr;
722 unsigned long c;
723 void *epc_virt;
724 int ret;
725
726 memset(&secinfo, 0, sizeof(secinfo));
727 secinfo.flags = modp->permissions & SGX_SECINFO_PERMISSION_MASK;
728
729 for (c = 0 ; c < modp->length; c += PAGE_SIZE) {
730 addr = encl->base + modp->offset + c;
731
732 sgx_reclaim_direct();
733
734 mutex_lock(&encl->lock);
735
736 entry = sgx_encl_load_page(encl, addr);
737 if (IS_ERR(entry)) {
738 ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
739 goto out_unlock;
740 }
741
742 /*
743 * Changing EPCM permissions is only supported on regular
744 * SGX pages. Attempting this change on other pages will
745 * result in #PF.
746 */
747 if (entry->type != SGX_PAGE_TYPE_REG) {
748 ret = -EINVAL;
749 goto out_unlock;
750 }
751
752 /*
753 * Apart from ensuring that read-access remains, do not verify
754 * the permission bits requested. Kernel has no control over
755 * how EPCM permissions can be relaxed from within the enclave.
756 * ENCLS[EMODPR] can only remove existing EPCM permissions,
757 * attempting to set new permissions will be ignored by the
758 * hardware.
759 */
760
761 /* Change EPCM permissions. */
762 epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
763 ret = __emodpr(&secinfo, epc_virt);
764 if (encls_faulted(ret)) {
765 /*
766 * All possible faults should be avoidable:
767 * parameters have been checked, will only change
768 * permissions of a regular page, and no concurrent
769 * SGX1/SGX2 ENCLS instructions since these
770 * are protected with mutex.
771 */
772 pr_err_once("EMODPR encountered exception %d\n",
773 ENCLS_TRAPNR(ret));
774 ret = -EFAULT;
775 goto out_unlock;
776 }
777 if (encls_failed(ret)) {
778 modp->result = ret;
779 ret = -EFAULT;
780 goto out_unlock;
781 }
782
783 ret = sgx_enclave_etrack(encl);
784 if (ret) {
785 ret = -EFAULT;
786 goto out_unlock;
787 }
788
789 mutex_unlock(&encl->lock);
790 }
791
792 ret = 0;
793 goto out;
794
795 out_unlock:
796 mutex_unlock(&encl->lock);
797 out:
798 modp->count = c;
799
800 return ret;
801 }
802
803 /**
804 * sgx_ioc_enclave_restrict_permissions() - handler for
805 * %SGX_IOC_ENCLAVE_RESTRICT_PERMISSIONS
806 * @encl: an enclave pointer
807 * @arg: userspace pointer to a &struct sgx_enclave_restrict_permissions
808 * instance
809 *
810 * SGX2 distinguishes between relaxing and restricting the enclave page
811 * permissions maintained by the hardware (EPCM permissions) of pages
812 * belonging to an initialized enclave (after SGX_IOC_ENCLAVE_INIT).
813 *
814 * EPCM permissions cannot be restricted from within the enclave, the enclave
815 * requires the kernel to run the privileged level 0 instructions ENCLS[EMODPR]
816 * and ENCLS[ETRACK]. An attempt to relax EPCM permissions with this call
817 * will be ignored by the hardware.
818 *
819 * Return:
820 * - 0: Success
821 * - -errno: Otherwise
822 */
sgx_ioc_enclave_restrict_permissions(struct sgx_encl * encl,void __user * arg)823 static long sgx_ioc_enclave_restrict_permissions(struct sgx_encl *encl,
824 void __user *arg)
825 {
826 struct sgx_enclave_restrict_permissions params;
827 long ret;
828
829 ret = sgx_ioc_sgx2_ready(encl);
830 if (ret)
831 return ret;
832
833 if (copy_from_user(¶ms, arg, sizeof(params)))
834 return -EFAULT;
835
836 if (sgx_validate_offset_length(encl, params.offset, params.length))
837 return -EINVAL;
838
839 if (params.permissions & ~SGX_SECINFO_PERMISSION_MASK)
840 return -EINVAL;
841
842 /*
843 * Fail early if invalid permissions requested to prevent ENCLS[EMODPR]
844 * from faulting later when the CPU does the same check.
845 */
846 if ((params.permissions & SGX_SECINFO_W) &&
847 !(params.permissions & SGX_SECINFO_R))
848 return -EINVAL;
849
850 if (params.result || params.count)
851 return -EINVAL;
852
853 ret = sgx_enclave_restrict_permissions(encl, ¶ms);
854
855 if (copy_to_user(arg, ¶ms, sizeof(params)))
856 return -EFAULT;
857
858 return ret;
859 }
860
861 /**
862 * sgx_enclave_modify_types() - Modify type of SGX enclave pages
863 * @encl: Enclave to which the pages belong.
864 * @modt: Checked parameters from user about which pages need modifying
865 * and their new page type.
866 *
867 * Return:
868 * - 0: Success
869 * - -errno: Otherwise
870 */
sgx_enclave_modify_types(struct sgx_encl * encl,struct sgx_enclave_modify_types * modt)871 static long sgx_enclave_modify_types(struct sgx_encl *encl,
872 struct sgx_enclave_modify_types *modt)
873 {
874 unsigned long max_prot_restore;
875 enum sgx_page_type page_type;
876 struct sgx_encl_page *entry;
877 struct sgx_secinfo secinfo;
878 unsigned long prot;
879 unsigned long addr;
880 unsigned long c;
881 void *epc_virt;
882 int ret;
883
884 page_type = modt->page_type & SGX_PAGE_TYPE_MASK;
885
886 /*
887 * The only new page types allowed by hardware are PT_TCS and PT_TRIM.
888 */
889 if (page_type != SGX_PAGE_TYPE_TCS && page_type != SGX_PAGE_TYPE_TRIM)
890 return -EINVAL;
891
892 memset(&secinfo, 0, sizeof(secinfo));
893
894 secinfo.flags = page_type << 8;
895
896 for (c = 0 ; c < modt->length; c += PAGE_SIZE) {
897 addr = encl->base + modt->offset + c;
898
899 sgx_reclaim_direct();
900
901 mutex_lock(&encl->lock);
902
903 entry = sgx_encl_load_page(encl, addr);
904 if (IS_ERR(entry)) {
905 ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
906 goto out_unlock;
907 }
908
909 /*
910 * Borrow the logic from the Intel SDM. Regular pages
911 * (SGX_PAGE_TYPE_REG) can change type to SGX_PAGE_TYPE_TCS
912 * or SGX_PAGE_TYPE_TRIM but TCS pages can only be trimmed.
913 * CET pages not supported yet.
914 */
915 if (!(entry->type == SGX_PAGE_TYPE_REG ||
916 (entry->type == SGX_PAGE_TYPE_TCS &&
917 page_type == SGX_PAGE_TYPE_TRIM))) {
918 ret = -EINVAL;
919 goto out_unlock;
920 }
921
922 max_prot_restore = entry->vm_max_prot_bits;
923
924 /*
925 * Once a regular page becomes a TCS page it cannot be
926 * changed back. So the maximum allowed protection reflects
927 * the TCS page that is always RW from kernel perspective but
928 * will be inaccessible from within enclave. Before doing
929 * so, do make sure that the new page type continues to
930 * respect the originally vetted page permissions.
931 */
932 if (entry->type == SGX_PAGE_TYPE_REG &&
933 page_type == SGX_PAGE_TYPE_TCS) {
934 if (~entry->vm_max_prot_bits & (VM_READ | VM_WRITE)) {
935 ret = -EPERM;
936 goto out_unlock;
937 }
938 prot = PROT_READ | PROT_WRITE;
939 entry->vm_max_prot_bits = calc_vm_prot_bits(prot, 0);
940
941 /*
942 * Prevent page from being reclaimed while mutex
943 * is released.
944 */
945 if (sgx_unmark_page_reclaimable(entry->epc_page)) {
946 ret = -EAGAIN;
947 goto out_entry_changed;
948 }
949
950 /*
951 * Do not keep encl->lock because of dependency on
952 * mmap_lock acquired in sgx_zap_enclave_ptes().
953 */
954 mutex_unlock(&encl->lock);
955
956 sgx_zap_enclave_ptes(encl, addr);
957
958 mutex_lock(&encl->lock);
959
960 sgx_mark_page_reclaimable(entry->epc_page);
961 }
962
963 /* Change EPC type */
964 epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
965 ret = __emodt(&secinfo, epc_virt);
966 if (encls_faulted(ret)) {
967 /*
968 * All possible faults should be avoidable:
969 * parameters have been checked, will only change
970 * valid page types, and no concurrent
971 * SGX1/SGX2 ENCLS instructions since these are
972 * protected with mutex.
973 */
974 pr_err_once("EMODT encountered exception %d\n",
975 ENCLS_TRAPNR(ret));
976 ret = -EFAULT;
977 goto out_entry_changed;
978 }
979 if (encls_failed(ret)) {
980 modt->result = ret;
981 ret = -EFAULT;
982 goto out_entry_changed;
983 }
984
985 ret = sgx_enclave_etrack(encl);
986 if (ret) {
987 ret = -EFAULT;
988 goto out_unlock;
989 }
990
991 entry->type = page_type;
992
993 mutex_unlock(&encl->lock);
994 }
995
996 ret = 0;
997 goto out;
998
999 out_entry_changed:
1000 entry->vm_max_prot_bits = max_prot_restore;
1001 out_unlock:
1002 mutex_unlock(&encl->lock);
1003 out:
1004 modt->count = c;
1005
1006 return ret;
1007 }
1008
1009 /**
1010 * sgx_ioc_enclave_modify_types() - handler for %SGX_IOC_ENCLAVE_MODIFY_TYPES
1011 * @encl: an enclave pointer
1012 * @arg: userspace pointer to a &struct sgx_enclave_modify_types instance
1013 *
1014 * Ability to change the enclave page type supports the following use cases:
1015 *
1016 * * It is possible to add TCS pages to an enclave by changing the type of
1017 * regular pages (%SGX_PAGE_TYPE_REG) to TCS (%SGX_PAGE_TYPE_TCS) pages.
1018 * With this support the number of threads supported by an initialized
1019 * enclave can be increased dynamically.
1020 *
1021 * * Regular or TCS pages can dynamically be removed from an initialized
1022 * enclave by changing the page type to %SGX_PAGE_TYPE_TRIM. Changing the
1023 * page type to %SGX_PAGE_TYPE_TRIM marks the page for removal with actual
1024 * removal done by handler of %SGX_IOC_ENCLAVE_REMOVE_PAGES ioctl() called
1025 * after ENCLU[EACCEPT] is run on %SGX_PAGE_TYPE_TRIM page from within the
1026 * enclave.
1027 *
1028 * Return:
1029 * - 0: Success
1030 * - -errno: Otherwise
1031 */
sgx_ioc_enclave_modify_types(struct sgx_encl * encl,void __user * arg)1032 static long sgx_ioc_enclave_modify_types(struct sgx_encl *encl,
1033 void __user *arg)
1034 {
1035 struct sgx_enclave_modify_types params;
1036 long ret;
1037
1038 ret = sgx_ioc_sgx2_ready(encl);
1039 if (ret)
1040 return ret;
1041
1042 if (copy_from_user(¶ms, arg, sizeof(params)))
1043 return -EFAULT;
1044
1045 if (sgx_validate_offset_length(encl, params.offset, params.length))
1046 return -EINVAL;
1047
1048 if (params.page_type & ~SGX_PAGE_TYPE_MASK)
1049 return -EINVAL;
1050
1051 if (params.result || params.count)
1052 return -EINVAL;
1053
1054 ret = sgx_enclave_modify_types(encl, ¶ms);
1055
1056 if (copy_to_user(arg, ¶ms, sizeof(params)))
1057 return -EFAULT;
1058
1059 return ret;
1060 }
1061
1062 /**
1063 * sgx_encl_remove_pages() - Remove trimmed pages from SGX enclave
1064 * @encl: Enclave to which the pages belong
1065 * @params: Checked parameters from user on which pages need to be removed
1066 *
1067 * Return:
1068 * - 0: Success.
1069 * - -errno: Otherwise.
1070 */
sgx_encl_remove_pages(struct sgx_encl * encl,struct sgx_enclave_remove_pages * params)1071 static long sgx_encl_remove_pages(struct sgx_encl *encl,
1072 struct sgx_enclave_remove_pages *params)
1073 {
1074 struct sgx_encl_page *entry;
1075 struct sgx_secinfo secinfo;
1076 unsigned long addr;
1077 unsigned long c;
1078 void *epc_virt;
1079 int ret;
1080
1081 memset(&secinfo, 0, sizeof(secinfo));
1082 secinfo.flags = SGX_SECINFO_R | SGX_SECINFO_W | SGX_SECINFO_X;
1083
1084 for (c = 0 ; c < params->length; c += PAGE_SIZE) {
1085 addr = encl->base + params->offset + c;
1086
1087 sgx_reclaim_direct();
1088
1089 mutex_lock(&encl->lock);
1090
1091 entry = sgx_encl_load_page(encl, addr);
1092 if (IS_ERR(entry)) {
1093 ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
1094 goto out_unlock;
1095 }
1096
1097 if (entry->type != SGX_PAGE_TYPE_TRIM) {
1098 ret = -EPERM;
1099 goto out_unlock;
1100 }
1101
1102 /*
1103 * ENCLS[EMODPR] is a no-op instruction used to inform if
1104 * ENCLU[EACCEPT] was run from within the enclave. If
1105 * ENCLS[EMODPR] is run with RWX on a trimmed page that is
1106 * not yet accepted then it will return
1107 * %SGX_PAGE_NOT_MODIFIABLE, after the trimmed page is
1108 * accepted the instruction will encounter a page fault.
1109 */
1110 epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
1111 ret = __emodpr(&secinfo, epc_virt);
1112 if (!encls_faulted(ret) || ENCLS_TRAPNR(ret) != X86_TRAP_PF) {
1113 ret = -EPERM;
1114 goto out_unlock;
1115 }
1116
1117 if (sgx_unmark_page_reclaimable(entry->epc_page)) {
1118 ret = -EBUSY;
1119 goto out_unlock;
1120 }
1121
1122 /*
1123 * Do not keep encl->lock because of dependency on
1124 * mmap_lock acquired in sgx_zap_enclave_ptes().
1125 */
1126 mutex_unlock(&encl->lock);
1127
1128 sgx_zap_enclave_ptes(encl, addr);
1129
1130 mutex_lock(&encl->lock);
1131
1132 sgx_encl_free_epc_page(entry->epc_page);
1133 encl->secs_child_cnt--;
1134 entry->epc_page = NULL;
1135 xa_erase(&encl->page_array, PFN_DOWN(entry->desc));
1136 sgx_encl_shrink(encl, NULL);
1137 kfree(entry);
1138
1139 mutex_unlock(&encl->lock);
1140 }
1141
1142 ret = 0;
1143 goto out;
1144
1145 out_unlock:
1146 mutex_unlock(&encl->lock);
1147 out:
1148 params->count = c;
1149
1150 return ret;
1151 }
1152
1153 /**
1154 * sgx_ioc_enclave_remove_pages() - handler for %SGX_IOC_ENCLAVE_REMOVE_PAGES
1155 * @encl: an enclave pointer
1156 * @arg: userspace pointer to &struct sgx_enclave_remove_pages instance
1157 *
1158 * Final step of the flow removing pages from an initialized enclave. The
1159 * complete flow is:
1160 *
1161 * 1) User changes the type of the pages to be removed to %SGX_PAGE_TYPE_TRIM
1162 * using the %SGX_IOC_ENCLAVE_MODIFY_TYPES ioctl().
1163 * 2) User approves the page removal by running ENCLU[EACCEPT] from within
1164 * the enclave.
1165 * 3) User initiates actual page removal using the
1166 * %SGX_IOC_ENCLAVE_REMOVE_PAGES ioctl() that is handled here.
1167 *
1168 * First remove any page table entries pointing to the page and then proceed
1169 * with the actual removal of the enclave page and data in support of it.
1170 *
1171 * VA pages are not affected by this removal. It is thus possible that the
1172 * enclave may end up with more VA pages than needed to support all its
1173 * pages.
1174 *
1175 * Return:
1176 * - 0: Success
1177 * - -errno: Otherwise
1178 */
sgx_ioc_enclave_remove_pages(struct sgx_encl * encl,void __user * arg)1179 static long sgx_ioc_enclave_remove_pages(struct sgx_encl *encl,
1180 void __user *arg)
1181 {
1182 struct sgx_enclave_remove_pages params;
1183 long ret;
1184
1185 ret = sgx_ioc_sgx2_ready(encl);
1186 if (ret)
1187 return ret;
1188
1189 if (copy_from_user(¶ms, arg, sizeof(params)))
1190 return -EFAULT;
1191
1192 if (sgx_validate_offset_length(encl, params.offset, params.length))
1193 return -EINVAL;
1194
1195 if (params.count)
1196 return -EINVAL;
1197
1198 ret = sgx_encl_remove_pages(encl, ¶ms);
1199
1200 if (copy_to_user(arg, ¶ms, sizeof(params)))
1201 return -EFAULT;
1202
1203 return ret;
1204 }
1205
sgx_ioctl(struct file * filep,unsigned int cmd,unsigned long arg)1206 long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1207 {
1208 struct sgx_encl *encl = filep->private_data;
1209 int ret;
1210
1211 if (test_and_set_bit(SGX_ENCL_IOCTL, &encl->flags))
1212 return -EBUSY;
1213
1214 switch (cmd) {
1215 case SGX_IOC_ENCLAVE_CREATE:
1216 ret = sgx_ioc_enclave_create(encl, (void __user *)arg);
1217 break;
1218 case SGX_IOC_ENCLAVE_ADD_PAGES:
1219 ret = sgx_ioc_enclave_add_pages(encl, (void __user *)arg);
1220 break;
1221 case SGX_IOC_ENCLAVE_INIT:
1222 ret = sgx_ioc_enclave_init(encl, (void __user *)arg);
1223 break;
1224 case SGX_IOC_ENCLAVE_PROVISION:
1225 ret = sgx_ioc_enclave_provision(encl, (void __user *)arg);
1226 break;
1227 case SGX_IOC_ENCLAVE_RESTRICT_PERMISSIONS:
1228 ret = sgx_ioc_enclave_restrict_permissions(encl,
1229 (void __user *)arg);
1230 break;
1231 case SGX_IOC_ENCLAVE_MODIFY_TYPES:
1232 ret = sgx_ioc_enclave_modify_types(encl, (void __user *)arg);
1233 break;
1234 case SGX_IOC_ENCLAVE_REMOVE_PAGES:
1235 ret = sgx_ioc_enclave_remove_pages(encl, (void __user *)arg);
1236 break;
1237 default:
1238 ret = -ENOIOCTLCMD;
1239 break;
1240 }
1241
1242 clear_bit(SGX_ENCL_IOCTL, &encl->flags);
1243 return ret;
1244 }
1245