1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2016-20 Intel Corporation. */ 3 4 #include <linux/file.h> 5 #include <linux/freezer.h> 6 #include <linux/highmem.h> 7 #include <linux/kthread.h> 8 #include <linux/miscdevice.h> 9 #include <linux/node.h> 10 #include <linux/pagemap.h> 11 #include <linux/ratelimit.h> 12 #include <linux/sched/mm.h> 13 #include <linux/sched/signal.h> 14 #include <linux/slab.h> 15 #include <linux/sysfs.h> 16 #include <linux/vmalloc.h> 17 #include <asm/sgx.h> 18 #include "driver.h" 19 #include "encl.h" 20 #include "encls.h" 21 22 struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; 23 static int sgx_nr_epc_sections; 24 static struct task_struct *ksgxd_tsk; 25 static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq); 26 static DEFINE_XARRAY(sgx_epc_address_space); 27 28 /* 29 * These variables are part of the state of the reclaimer, and must be accessed 30 * with sgx_reclaimer_lock acquired. 31 */ 32 static LIST_HEAD(sgx_active_page_list); 33 static DEFINE_SPINLOCK(sgx_reclaimer_lock); 34 35 static atomic_long_t sgx_nr_free_pages = ATOMIC_LONG_INIT(0); 36 37 /* Nodes with one or more EPC sections. */ 38 static nodemask_t sgx_numa_mask; 39 40 /* 41 * Array with one list_head for each possible NUMA node. Each 42 * list contains all the sgx_epc_section's which are on that 43 * node. 44 */ 45 static struct sgx_numa_node *sgx_numa_nodes; 46 47 static LIST_HEAD(sgx_dirty_page_list); 48 49 /* 50 * Reset post-kexec EPC pages to the uninitialized state. The pages are removed 51 * from the input list, and made available for the page allocator. SECS pages 52 * prepending their children in the input list are left intact. 53 * 54 * Return 0 when sanitization was successful or kthread was stopped, and the 55 * number of unsanitized pages otherwise. 56 */ 57 static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list) 58 { 59 unsigned long left_dirty = 0; 60 struct sgx_epc_page *page; 61 LIST_HEAD(dirty); 62 int ret; 63 64 /* dirty_page_list is thread-local, no need for a lock: */ 65 while (!list_empty(dirty_page_list)) { 66 if (kthread_should_stop()) 67 return 0; 68 69 page = list_first_entry(dirty_page_list, struct sgx_epc_page, list); 70 71 /* 72 * Checking page->poison without holding the node->lock 73 * is racy, but losing the race (i.e. poison is set just 74 * after the check) just means __eremove() will be uselessly 75 * called for a page that sgx_free_epc_page() will put onto 76 * the node->sgx_poison_page_list later. 77 */ 78 if (page->poison) { 79 struct sgx_epc_section *section = &sgx_epc_sections[page->section]; 80 struct sgx_numa_node *node = section->node; 81 82 spin_lock(&node->lock); 83 list_move(&page->list, &node->sgx_poison_page_list); 84 spin_unlock(&node->lock); 85 86 continue; 87 } 88 89 ret = __eremove(sgx_get_epc_virt_addr(page)); 90 if (!ret) { 91 /* 92 * page is now sanitized. Make it available via the SGX 93 * page allocator: 94 */ 95 list_del(&page->list); 96 sgx_free_epc_page(page); 97 } else { 98 /* The page is not yet clean - move to the dirty list. */ 99 list_move_tail(&page->list, &dirty); 100 left_dirty++; 101 } 102 103 cond_resched(); 104 } 105 106 list_splice(&dirty, dirty_page_list); 107 return left_dirty; 108 } 109 110 static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page) 111 { 112 struct sgx_encl_page *page = epc_page->owner; 113 struct sgx_encl *encl = page->encl; 114 struct sgx_encl_mm *encl_mm; 115 bool ret = true; 116 int idx; 117 118 idx = srcu_read_lock(&encl->srcu); 119 120 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { 121 if (!mmget_not_zero(encl_mm->mm)) 122 continue; 123 124 mmap_read_lock(encl_mm->mm); 125 ret = !sgx_encl_test_and_clear_young(encl_mm->mm, page); 126 mmap_read_unlock(encl_mm->mm); 127 128 mmput_async(encl_mm->mm); 129 130 if (!ret) 131 break; 132 } 133 134 srcu_read_unlock(&encl->srcu, idx); 135 136 if (!ret) 137 return false; 138 139 return true; 140 } 141 142 static void sgx_reclaimer_block(struct sgx_epc_page *epc_page) 143 { 144 struct sgx_encl_page *page = epc_page->owner; 145 unsigned long addr = page->desc & PAGE_MASK; 146 struct sgx_encl *encl = page->encl; 147 int ret; 148 149 sgx_zap_enclave_ptes(encl, addr); 150 151 mutex_lock(&encl->lock); 152 153 ret = __eblock(sgx_get_epc_virt_addr(epc_page)); 154 if (encls_failed(ret)) 155 ENCLS_WARN(ret, "EBLOCK"); 156 157 mutex_unlock(&encl->lock); 158 } 159 160 static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot, 161 struct sgx_backing *backing) 162 { 163 struct sgx_pageinfo pginfo; 164 int ret; 165 166 pginfo.addr = 0; 167 pginfo.secs = 0; 168 169 pginfo.contents = (unsigned long)kmap_local_page(backing->contents); 170 pginfo.metadata = (unsigned long)kmap_local_page(backing->pcmd) + 171 backing->pcmd_offset; 172 173 ret = __ewb(&pginfo, sgx_get_epc_virt_addr(epc_page), va_slot); 174 set_page_dirty(backing->pcmd); 175 set_page_dirty(backing->contents); 176 177 kunmap_local((void *)(unsigned long)(pginfo.metadata - 178 backing->pcmd_offset)); 179 kunmap_local((void *)(unsigned long)pginfo.contents); 180 181 return ret; 182 } 183 184 void sgx_ipi_cb(void *info) 185 { 186 } 187 188 /* 189 * Swap page to the regular memory transformed to the blocked state by using 190 * EBLOCK, which means that it can no longer be referenced (no new TLB entries). 191 * 192 * The first trial just tries to write the page assuming that some other thread 193 * has reset the count for threads inside the enclave by using ETRACK, and 194 * previous thread count has been zeroed out. The second trial calls ETRACK 195 * before EWB. If that fails we kick all the HW threads out, and then do EWB, 196 * which should be guaranteed the succeed. 197 */ 198 static void sgx_encl_ewb(struct sgx_epc_page *epc_page, 199 struct sgx_backing *backing) 200 { 201 struct sgx_encl_page *encl_page = epc_page->owner; 202 struct sgx_encl *encl = encl_page->encl; 203 struct sgx_va_page *va_page; 204 unsigned int va_offset; 205 void *va_slot; 206 int ret; 207 208 encl_page->desc &= ~SGX_ENCL_PAGE_BEING_RECLAIMED; 209 210 va_page = list_first_entry(&encl->va_pages, struct sgx_va_page, 211 list); 212 va_offset = sgx_alloc_va_slot(va_page); 213 va_slot = sgx_get_epc_virt_addr(va_page->epc_page) + va_offset; 214 if (sgx_va_page_full(va_page)) 215 list_move_tail(&va_page->list, &encl->va_pages); 216 217 ret = __sgx_encl_ewb(epc_page, va_slot, backing); 218 if (ret == SGX_NOT_TRACKED) { 219 ret = __etrack(sgx_get_epc_virt_addr(encl->secs.epc_page)); 220 if (ret) { 221 if (encls_failed(ret)) 222 ENCLS_WARN(ret, "ETRACK"); 223 } 224 225 ret = __sgx_encl_ewb(epc_page, va_slot, backing); 226 if (ret == SGX_NOT_TRACKED) { 227 /* 228 * Slow path, send IPIs to kick cpus out of the 229 * enclave. Note, it's imperative that the cpu 230 * mask is generated *after* ETRACK, else we'll 231 * miss cpus that entered the enclave between 232 * generating the mask and incrementing epoch. 233 */ 234 on_each_cpu_mask(sgx_encl_cpumask(encl), 235 sgx_ipi_cb, NULL, 1); 236 ret = __sgx_encl_ewb(epc_page, va_slot, backing); 237 } 238 } 239 240 if (ret) { 241 if (encls_failed(ret)) 242 ENCLS_WARN(ret, "EWB"); 243 244 sgx_free_va_slot(va_page, va_offset); 245 } else { 246 encl_page->desc |= va_offset; 247 encl_page->va_page = va_page; 248 } 249 } 250 251 static void sgx_reclaimer_write(struct sgx_epc_page *epc_page, 252 struct sgx_backing *backing) 253 { 254 struct sgx_encl_page *encl_page = epc_page->owner; 255 struct sgx_encl *encl = encl_page->encl; 256 struct sgx_backing secs_backing; 257 int ret; 258 259 mutex_lock(&encl->lock); 260 261 sgx_encl_ewb(epc_page, backing); 262 encl_page->epc_page = NULL; 263 encl->secs_child_cnt--; 264 sgx_encl_put_backing(backing); 265 266 if (!encl->secs_child_cnt && test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) { 267 ret = sgx_encl_alloc_backing(encl, PFN_DOWN(encl->size), 268 &secs_backing); 269 if (ret) 270 goto out; 271 272 sgx_encl_ewb(encl->secs.epc_page, &secs_backing); 273 274 sgx_encl_free_epc_page(encl->secs.epc_page); 275 encl->secs.epc_page = NULL; 276 277 sgx_encl_put_backing(&secs_backing); 278 } 279 280 out: 281 mutex_unlock(&encl->lock); 282 } 283 284 /* 285 * Take a fixed number of pages from the head of the active page pool and 286 * reclaim them to the enclave's private shmem files. Skip the pages, which have 287 * been accessed since the last scan. Move those pages to the tail of active 288 * page pool so that the pages get scanned in LRU like fashion. 289 * 290 * Batch process a chunk of pages (at the moment 16) in order to degrade amount 291 * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit 292 * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI 293 * + EWB) but not sufficiently. Reclaiming one page at a time would also be 294 * problematic as it would increase the lock contention too much, which would 295 * halt forward progress. 296 */ 297 static void sgx_reclaim_pages(void) 298 { 299 struct sgx_epc_page *chunk[SGX_NR_TO_SCAN]; 300 struct sgx_backing backing[SGX_NR_TO_SCAN]; 301 struct sgx_encl_page *encl_page; 302 struct sgx_epc_page *epc_page; 303 pgoff_t page_index; 304 int cnt = 0; 305 int ret; 306 int i; 307 308 spin_lock(&sgx_reclaimer_lock); 309 for (i = 0; i < SGX_NR_TO_SCAN; i++) { 310 if (list_empty(&sgx_active_page_list)) 311 break; 312 313 epc_page = list_first_entry(&sgx_active_page_list, 314 struct sgx_epc_page, list); 315 list_del_init(&epc_page->list); 316 encl_page = epc_page->owner; 317 318 if (kref_get_unless_zero(&encl_page->encl->refcount) != 0) 319 chunk[cnt++] = epc_page; 320 else 321 /* The owner is freeing the page. No need to add the 322 * page back to the list of reclaimable pages. 323 */ 324 epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; 325 } 326 spin_unlock(&sgx_reclaimer_lock); 327 328 for (i = 0; i < cnt; i++) { 329 epc_page = chunk[i]; 330 encl_page = epc_page->owner; 331 332 if (!sgx_reclaimer_age(epc_page)) 333 goto skip; 334 335 page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base); 336 337 mutex_lock(&encl_page->encl->lock); 338 ret = sgx_encl_alloc_backing(encl_page->encl, page_index, &backing[i]); 339 if (ret) { 340 mutex_unlock(&encl_page->encl->lock); 341 goto skip; 342 } 343 344 encl_page->desc |= SGX_ENCL_PAGE_BEING_RECLAIMED; 345 mutex_unlock(&encl_page->encl->lock); 346 continue; 347 348 skip: 349 spin_lock(&sgx_reclaimer_lock); 350 list_add_tail(&epc_page->list, &sgx_active_page_list); 351 spin_unlock(&sgx_reclaimer_lock); 352 353 kref_put(&encl_page->encl->refcount, sgx_encl_release); 354 355 chunk[i] = NULL; 356 } 357 358 for (i = 0; i < cnt; i++) { 359 epc_page = chunk[i]; 360 if (epc_page) 361 sgx_reclaimer_block(epc_page); 362 } 363 364 for (i = 0; i < cnt; i++) { 365 epc_page = chunk[i]; 366 if (!epc_page) 367 continue; 368 369 encl_page = epc_page->owner; 370 sgx_reclaimer_write(epc_page, &backing[i]); 371 372 kref_put(&encl_page->encl->refcount, sgx_encl_release); 373 epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; 374 375 sgx_free_epc_page(epc_page); 376 } 377 } 378 379 static bool sgx_should_reclaim(unsigned long watermark) 380 { 381 return atomic_long_read(&sgx_nr_free_pages) < watermark && 382 !list_empty(&sgx_active_page_list); 383 } 384 385 /* 386 * sgx_reclaim_direct() should be called (without enclave's mutex held) 387 * in locations where SGX memory resources might be low and might be 388 * needed in order to make forward progress. 389 */ 390 void sgx_reclaim_direct(void) 391 { 392 if (sgx_should_reclaim(SGX_NR_LOW_PAGES)) 393 sgx_reclaim_pages(); 394 } 395 396 static int ksgxd(void *p) 397 { 398 set_freezable(); 399 400 /* 401 * Sanitize pages in order to recover from kexec(). The 2nd pass is 402 * required for SECS pages, whose child pages blocked EREMOVE. 403 */ 404 __sgx_sanitize_pages(&sgx_dirty_page_list); 405 WARN_ON(__sgx_sanitize_pages(&sgx_dirty_page_list)); 406 407 while (!kthread_should_stop()) { 408 if (try_to_freeze()) 409 continue; 410 411 wait_event_freezable(ksgxd_waitq, 412 kthread_should_stop() || 413 sgx_should_reclaim(SGX_NR_HIGH_PAGES)); 414 415 if (sgx_should_reclaim(SGX_NR_HIGH_PAGES)) 416 sgx_reclaim_pages(); 417 418 cond_resched(); 419 } 420 421 return 0; 422 } 423 424 static bool __init sgx_page_reclaimer_init(void) 425 { 426 struct task_struct *tsk; 427 428 tsk = kthread_run(ksgxd, NULL, "ksgxd"); 429 if (IS_ERR(tsk)) 430 return false; 431 432 ksgxd_tsk = tsk; 433 434 return true; 435 } 436 437 bool current_is_ksgxd(void) 438 { 439 return current == ksgxd_tsk; 440 } 441 442 static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid) 443 { 444 struct sgx_numa_node *node = &sgx_numa_nodes[nid]; 445 struct sgx_epc_page *page = NULL; 446 447 spin_lock(&node->lock); 448 449 if (list_empty(&node->free_page_list)) { 450 spin_unlock(&node->lock); 451 return NULL; 452 } 453 454 page = list_first_entry(&node->free_page_list, struct sgx_epc_page, list); 455 list_del_init(&page->list); 456 page->flags = 0; 457 458 spin_unlock(&node->lock); 459 atomic_long_dec(&sgx_nr_free_pages); 460 461 return page; 462 } 463 464 /** 465 * __sgx_alloc_epc_page() - Allocate an EPC page 466 * 467 * Iterate through NUMA nodes and reserve ia free EPC page to the caller. Start 468 * from the NUMA node, where the caller is executing. 469 * 470 * Return: 471 * - an EPC page: A borrowed EPC pages were available. 472 * - NULL: Out of EPC pages. 473 */ 474 struct sgx_epc_page *__sgx_alloc_epc_page(void) 475 { 476 struct sgx_epc_page *page; 477 int nid_of_current = numa_node_id(); 478 int nid_start, nid; 479 480 /* 481 * Try local node first. If it doesn't have an EPC section, 482 * fall back to the non-local NUMA nodes. 483 */ 484 if (node_isset(nid_of_current, sgx_numa_mask)) 485 nid_start = nid_of_current; 486 else 487 nid_start = next_node_in(nid_of_current, sgx_numa_mask); 488 489 nid = nid_start; 490 do { 491 page = __sgx_alloc_epc_page_from_node(nid); 492 if (page) 493 return page; 494 495 nid = next_node_in(nid, sgx_numa_mask); 496 } while (nid != nid_start); 497 498 return ERR_PTR(-ENOMEM); 499 } 500 501 /** 502 * sgx_mark_page_reclaimable() - Mark a page as reclaimable 503 * @page: EPC page 504 * 505 * Mark a page as reclaimable and add it to the active page list. Pages 506 * are automatically removed from the active list when freed. 507 */ 508 void sgx_mark_page_reclaimable(struct sgx_epc_page *page) 509 { 510 spin_lock(&sgx_reclaimer_lock); 511 page->flags |= SGX_EPC_PAGE_RECLAIMER_TRACKED; 512 list_add_tail(&page->list, &sgx_active_page_list); 513 spin_unlock(&sgx_reclaimer_lock); 514 } 515 516 /** 517 * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list 518 * @page: EPC page 519 * 520 * Clear the reclaimable flag and remove the page from the active page list. 521 * 522 * Return: 523 * 0 on success, 524 * -EBUSY if the page is in the process of being reclaimed 525 */ 526 int sgx_unmark_page_reclaimable(struct sgx_epc_page *page) 527 { 528 spin_lock(&sgx_reclaimer_lock); 529 if (page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED) { 530 /* The page is being reclaimed. */ 531 if (list_empty(&page->list)) { 532 spin_unlock(&sgx_reclaimer_lock); 533 return -EBUSY; 534 } 535 536 list_del(&page->list); 537 page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; 538 } 539 spin_unlock(&sgx_reclaimer_lock); 540 541 return 0; 542 } 543 544 /** 545 * sgx_alloc_epc_page() - Allocate an EPC page 546 * @owner: the owner of the EPC page 547 * @reclaim: reclaim pages if necessary 548 * 549 * Iterate through EPC sections and borrow a free EPC page to the caller. When a 550 * page is no longer needed it must be released with sgx_free_epc_page(). If 551 * @reclaim is set to true, directly reclaim pages when we are out of pages. No 552 * mm's can be locked when @reclaim is set to true. 553 * 554 * Finally, wake up ksgxd when the number of pages goes below the watermark 555 * before returning back to the caller. 556 * 557 * Return: 558 * an EPC page, 559 * -errno on error 560 */ 561 struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim) 562 { 563 struct sgx_epc_page *page; 564 565 for ( ; ; ) { 566 page = __sgx_alloc_epc_page(); 567 if (!IS_ERR(page)) { 568 page->owner = owner; 569 break; 570 } 571 572 if (list_empty(&sgx_active_page_list)) 573 return ERR_PTR(-ENOMEM); 574 575 if (!reclaim) { 576 page = ERR_PTR(-EBUSY); 577 break; 578 } 579 580 if (signal_pending(current)) { 581 page = ERR_PTR(-ERESTARTSYS); 582 break; 583 } 584 585 sgx_reclaim_pages(); 586 cond_resched(); 587 } 588 589 if (sgx_should_reclaim(SGX_NR_LOW_PAGES)) 590 wake_up(&ksgxd_waitq); 591 592 return page; 593 } 594 595 /** 596 * sgx_free_epc_page() - Free an EPC page 597 * @page: an EPC page 598 * 599 * Put the EPC page back to the list of free pages. It's the caller's 600 * responsibility to make sure that the page is in uninitialized state. In other 601 * words, do EREMOVE, EWB or whatever operation is necessary before calling 602 * this function. 603 */ 604 void sgx_free_epc_page(struct sgx_epc_page *page) 605 { 606 struct sgx_epc_section *section = &sgx_epc_sections[page->section]; 607 struct sgx_numa_node *node = section->node; 608 609 spin_lock(&node->lock); 610 611 page->owner = NULL; 612 if (page->poison) 613 list_add(&page->list, &node->sgx_poison_page_list); 614 else 615 list_add_tail(&page->list, &node->free_page_list); 616 page->flags = SGX_EPC_PAGE_IS_FREE; 617 618 spin_unlock(&node->lock); 619 atomic_long_inc(&sgx_nr_free_pages); 620 } 621 622 static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, 623 unsigned long index, 624 struct sgx_epc_section *section) 625 { 626 unsigned long nr_pages = size >> PAGE_SHIFT; 627 unsigned long i; 628 629 section->virt_addr = memremap(phys_addr, size, MEMREMAP_WB); 630 if (!section->virt_addr) 631 return false; 632 633 section->pages = vmalloc(nr_pages * sizeof(struct sgx_epc_page)); 634 if (!section->pages) { 635 memunmap(section->virt_addr); 636 return false; 637 } 638 639 section->phys_addr = phys_addr; 640 xa_store_range(&sgx_epc_address_space, section->phys_addr, 641 phys_addr + size - 1, section, GFP_KERNEL); 642 643 for (i = 0; i < nr_pages; i++) { 644 section->pages[i].section = index; 645 section->pages[i].flags = 0; 646 section->pages[i].owner = NULL; 647 section->pages[i].poison = 0; 648 list_add_tail(§ion->pages[i].list, &sgx_dirty_page_list); 649 } 650 651 return true; 652 } 653 654 bool arch_is_platform_page(u64 paddr) 655 { 656 return !!xa_load(&sgx_epc_address_space, paddr); 657 } 658 EXPORT_SYMBOL_GPL(arch_is_platform_page); 659 660 static struct sgx_epc_page *sgx_paddr_to_page(u64 paddr) 661 { 662 struct sgx_epc_section *section; 663 664 section = xa_load(&sgx_epc_address_space, paddr); 665 if (!section) 666 return NULL; 667 668 return §ion->pages[PFN_DOWN(paddr - section->phys_addr)]; 669 } 670 671 /* 672 * Called in process context to handle a hardware reported 673 * error in an SGX EPC page. 674 * If the MF_ACTION_REQUIRED bit is set in flags, then the 675 * context is the task that consumed the poison data. Otherwise 676 * this is called from a kernel thread unrelated to the page. 677 */ 678 int arch_memory_failure(unsigned long pfn, int flags) 679 { 680 struct sgx_epc_page *page = sgx_paddr_to_page(pfn << PAGE_SHIFT); 681 struct sgx_epc_section *section; 682 struct sgx_numa_node *node; 683 684 /* 685 * mm/memory-failure.c calls this routine for all errors 686 * where there isn't a "struct page" for the address. But that 687 * includes other address ranges besides SGX. 688 */ 689 if (!page) 690 return -ENXIO; 691 692 /* 693 * If poison was consumed synchronously. Send a SIGBUS to 694 * the task. Hardware has already exited the SGX enclave and 695 * will not allow re-entry to an enclave that has a memory 696 * error. The signal may help the task understand why the 697 * enclave is broken. 698 */ 699 if (flags & MF_ACTION_REQUIRED) 700 force_sig(SIGBUS); 701 702 section = &sgx_epc_sections[page->section]; 703 node = section->node; 704 705 spin_lock(&node->lock); 706 707 /* Already poisoned? Nothing more to do */ 708 if (page->poison) 709 goto out; 710 711 page->poison = 1; 712 713 /* 714 * If the page is on a free list, move it to the per-node 715 * poison page list. 716 */ 717 if (page->flags & SGX_EPC_PAGE_IS_FREE) { 718 list_move(&page->list, &node->sgx_poison_page_list); 719 goto out; 720 } 721 722 /* 723 * TBD: Add additional plumbing to enable pre-emptive 724 * action for asynchronous poison notification. Until 725 * then just hope that the poison: 726 * a) is not accessed - sgx_free_epc_page() will deal with it 727 * when the user gives it back 728 * b) results in a recoverable machine check rather than 729 * a fatal one 730 */ 731 out: 732 spin_unlock(&node->lock); 733 return 0; 734 } 735 736 /* 737 * A section metric is concatenated in a way that @low bits 12-31 define the 738 * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the 739 * metric. 740 */ 741 static inline u64 __init sgx_calc_section_metric(u64 low, u64 high) 742 { 743 return (low & GENMASK_ULL(31, 12)) + 744 ((high & GENMASK_ULL(19, 0)) << 32); 745 } 746 747 #ifdef CONFIG_NUMA 748 static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf) 749 { 750 return sysfs_emit(buf, "%lu\n", sgx_numa_nodes[dev->id].size); 751 } 752 static DEVICE_ATTR_RO(sgx_total_bytes); 753 754 static umode_t arch_node_attr_is_visible(struct kobject *kobj, 755 struct attribute *attr, int idx) 756 { 757 /* Make all x86/ attributes invisible when SGX is not initialized: */ 758 if (nodes_empty(sgx_numa_mask)) 759 return 0; 760 761 return attr->mode; 762 } 763 764 static struct attribute *arch_node_dev_attrs[] = { 765 &dev_attr_sgx_total_bytes.attr, 766 NULL, 767 }; 768 769 const struct attribute_group arch_node_dev_group = { 770 .name = "x86", 771 .attrs = arch_node_dev_attrs, 772 .is_visible = arch_node_attr_is_visible, 773 }; 774 775 static void __init arch_update_sysfs_visibility(int nid) 776 { 777 struct node *node = node_devices[nid]; 778 int ret; 779 780 ret = sysfs_update_group(&node->dev.kobj, &arch_node_dev_group); 781 782 if (ret) 783 pr_err("sysfs update failed (%d), files may be invisible", ret); 784 } 785 #else /* !CONFIG_NUMA */ 786 static void __init arch_update_sysfs_visibility(int nid) {} 787 #endif 788 789 static bool __init sgx_page_cache_init(void) 790 { 791 u32 eax, ebx, ecx, edx, type; 792 u64 pa, size; 793 int nid; 794 int i; 795 796 sgx_numa_nodes = kmalloc_array(num_possible_nodes(), sizeof(*sgx_numa_nodes), GFP_KERNEL); 797 if (!sgx_numa_nodes) 798 return false; 799 800 for (i = 0; i < ARRAY_SIZE(sgx_epc_sections); i++) { 801 cpuid_count(SGX_CPUID, i + SGX_CPUID_EPC, &eax, &ebx, &ecx, &edx); 802 803 type = eax & SGX_CPUID_EPC_MASK; 804 if (type == SGX_CPUID_EPC_INVALID) 805 break; 806 807 if (type != SGX_CPUID_EPC_SECTION) { 808 pr_err_once("Unknown EPC section type: %u\n", type); 809 break; 810 } 811 812 pa = sgx_calc_section_metric(eax, ebx); 813 size = sgx_calc_section_metric(ecx, edx); 814 815 pr_info("EPC section 0x%llx-0x%llx\n", pa, pa + size - 1); 816 817 if (!sgx_setup_epc_section(pa, size, i, &sgx_epc_sections[i])) { 818 pr_err("No free memory for an EPC section\n"); 819 break; 820 } 821 822 nid = numa_map_to_online_node(phys_to_target_node(pa)); 823 if (nid == NUMA_NO_NODE) { 824 /* The physical address is already printed above. */ 825 pr_warn(FW_BUG "Unable to map EPC section to online node. Fallback to the NUMA node 0.\n"); 826 nid = 0; 827 } 828 829 if (!node_isset(nid, sgx_numa_mask)) { 830 spin_lock_init(&sgx_numa_nodes[nid].lock); 831 INIT_LIST_HEAD(&sgx_numa_nodes[nid].free_page_list); 832 INIT_LIST_HEAD(&sgx_numa_nodes[nid].sgx_poison_page_list); 833 node_set(nid, sgx_numa_mask); 834 sgx_numa_nodes[nid].size = 0; 835 836 /* Make SGX-specific node sysfs files visible: */ 837 arch_update_sysfs_visibility(nid); 838 } 839 840 sgx_epc_sections[i].node = &sgx_numa_nodes[nid]; 841 sgx_numa_nodes[nid].size += size; 842 843 sgx_nr_epc_sections++; 844 } 845 846 if (!sgx_nr_epc_sections) { 847 pr_err("There are zero EPC sections.\n"); 848 return false; 849 } 850 851 for_each_online_node(nid) { 852 if (!node_isset(nid, sgx_numa_mask) && 853 node_state(nid, N_MEMORY) && node_state(nid, N_CPU)) 854 pr_info("node%d has both CPUs and memory but doesn't have an EPC section\n", 855 nid); 856 } 857 858 return true; 859 } 860 861 /* 862 * Update the SGX_LEPUBKEYHASH MSRs to the values specified by caller. 863 * Bare-metal driver requires to update them to hash of enclave's signer 864 * before EINIT. KVM needs to update them to guest's virtual MSR values 865 * before doing EINIT from guest. 866 */ 867 void sgx_update_lepubkeyhash(u64 *lepubkeyhash) 868 { 869 int i; 870 871 WARN_ON_ONCE(preemptible()); 872 873 for (i = 0; i < 4; i++) 874 wrmsrl(MSR_IA32_SGXLEPUBKEYHASH0 + i, lepubkeyhash[i]); 875 } 876 877 const struct file_operations sgx_provision_fops = { 878 .owner = THIS_MODULE, 879 }; 880 881 static struct miscdevice sgx_dev_provision = { 882 .minor = MISC_DYNAMIC_MINOR, 883 .name = "sgx_provision", 884 .nodename = "sgx_provision", 885 .fops = &sgx_provision_fops, 886 }; 887 888 /** 889 * sgx_set_attribute() - Update allowed attributes given file descriptor 890 * @allowed_attributes: Pointer to allowed enclave attributes 891 * @attribute_fd: File descriptor for specific attribute 892 * 893 * Append enclave attribute indicated by file descriptor to allowed 894 * attributes. Currently only SGX_ATTR_PROVISIONKEY indicated by 895 * /dev/sgx_provision is supported. 896 * 897 * Return: 898 * -0: SGX_ATTR_PROVISIONKEY is appended to allowed_attributes 899 * -EINVAL: Invalid, or not supported file descriptor 900 */ 901 int sgx_set_attribute(unsigned long *allowed_attributes, 902 unsigned int attribute_fd) 903 { 904 struct fd f = fdget(attribute_fd); 905 906 if (!fd_file(f)) 907 return -EINVAL; 908 909 if (fd_file(f)->f_op != &sgx_provision_fops) { 910 fdput(f); 911 return -EINVAL; 912 } 913 914 *allowed_attributes |= SGX_ATTR_PROVISIONKEY; 915 916 fdput(f); 917 return 0; 918 } 919 EXPORT_SYMBOL_GPL(sgx_set_attribute); 920 921 static int __init sgx_init(void) 922 { 923 int ret; 924 int i; 925 926 if (!cpu_feature_enabled(X86_FEATURE_SGX)) 927 return -ENODEV; 928 929 if (!sgx_page_cache_init()) 930 return -ENOMEM; 931 932 if (!sgx_page_reclaimer_init()) { 933 ret = -ENOMEM; 934 goto err_page_cache; 935 } 936 937 ret = misc_register(&sgx_dev_provision); 938 if (ret) 939 goto err_kthread; 940 941 /* 942 * Always try to initialize the native *and* KVM drivers. 943 * The KVM driver is less picky than the native one and 944 * can function if the native one is not supported on the 945 * current system or fails to initialize. 946 * 947 * Error out only if both fail to initialize. 948 */ 949 ret = sgx_drv_init(); 950 951 if (sgx_vepc_init() && ret) 952 goto err_provision; 953 954 return 0; 955 956 err_provision: 957 misc_deregister(&sgx_dev_provision); 958 959 err_kthread: 960 kthread_stop(ksgxd_tsk); 961 962 err_page_cache: 963 for (i = 0; i < sgx_nr_epc_sections; i++) { 964 vfree(sgx_epc_sections[i].pages); 965 memunmap(sgx_epc_sections[i].virt_addr); 966 } 967 968 return ret; 969 } 970 971 device_initcall(sgx_init); 972