1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/gfp.h> 4 #include <linux/init.h> 5 #include <linux/mm.h> 6 #include <linux/time_namespace.h> 7 #include <linux/types.h> 8 #include <linux/vdso_datastore.h> 9 #include <vdso/datapage.h> 10 11 static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {}; 12 13 #ifdef CONFIG_GENERIC_GETTIMEOFDAY 14 struct vdso_time_data *vdso_k_time_data __ro_after_init = 15 (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE]; 16 17 static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE); 18 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */ 19 20 #ifdef CONFIG_VDSO_GETRANDOM 21 struct vdso_rng_data *vdso_k_rng_data __ro_after_init = 22 (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE]; 23 24 static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE); 25 #endif /* CONFIG_VDSO_GETRANDOM */ 26 27 #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA 28 struct vdso_arch_data *vdso_k_arch_data __ro_after_init = 29 (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE]; 30 #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */ 31 32 static struct page *vdso_data_pages __ro_after_init; 33 34 void __init vdso_setup_data_pages(void) 35 { 36 unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE); 37 38 /* 39 * Allocate the data pages dynamically. SPARC does not support mapping 40 * static pages to be mapped into userspace. 41 * It is also a requirement for mlockall() support. 42 * 43 * Do not use folios. In time namespaces the pages are mapped in a different order 44 * to userspace, which is not handled by the folio optimizations in finish_fault(). 45 */ 46 vdso_data_pages = alloc_pages(GFP_KERNEL, order); 47 if (!vdso_data_pages) 48 panic("Unable to allocate VDSO storage pages"); 49 50 /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */ 51 split_page(vdso_data_pages, order); 52 53 /* Move the data already written by other subsystems to the new pages */ 54 memcpy(page_address(vdso_data_pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE); 55 56 if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) 57 vdso_k_time_data = page_address(vdso_data_pages + VDSO_TIME_PAGE_OFFSET); 58 59 if (IS_ENABLED(CONFIG_VDSO_GETRANDOM)) 60 vdso_k_rng_data = page_address(vdso_data_pages + VDSO_RNG_PAGE_OFFSET); 61 62 if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) 63 vdso_k_arch_data = page_address(vdso_data_pages + VDSO_ARCH_PAGES_START); 64 } 65 66 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, 67 struct vm_area_struct *vma, struct vm_fault *vmf) 68 { 69 struct page *page, *timens_page; 70 71 page = vdso_data_pages + vmf->pgoff; 72 timens_page = find_timens_vvar_page(vma); 73 74 switch (vmf->pgoff) { 75 case VDSO_TIME_PAGE_OFFSET: 76 if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) 77 return VM_FAULT_SIGBUS; 78 if (timens_page) { 79 /* 80 * Fault in VVAR page too, since it will be accessed 81 * to get clock data anyway. 82 */ 83 unsigned long addr; 84 vm_fault_t err; 85 86 addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; 87 err = vmf_insert_page(vma, addr, page); 88 if (unlikely(err & VM_FAULT_ERROR)) 89 return err; 90 page = timens_page; 91 } 92 break; 93 case VDSO_TIMENS_PAGE_OFFSET: 94 /* 95 * If a task belongs to a time namespace then a namespace 96 * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and 97 * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET 98 * offset. 99 * See also the comment near timens_setup_vdso_data(). 100 */ 101 if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page) 102 return VM_FAULT_SIGBUS; 103 page = vdso_data_pages + VDSO_TIME_PAGE_OFFSET; 104 break; 105 case VDSO_RNG_PAGE_OFFSET: 106 if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM)) 107 return VM_FAULT_SIGBUS; 108 break; 109 case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END: 110 if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) 111 return VM_FAULT_SIGBUS; 112 break; 113 default: 114 return VM_FAULT_SIGBUS; 115 } 116 117 get_page(page); 118 vmf->page = page; 119 return 0; 120 } 121 122 const struct vm_special_mapping vdso_vvar_mapping = { 123 .name = "[vvar]", 124 .fault = vvar_fault, 125 }; 126 127 struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr) 128 { 129 return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, 130 VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | 131 VM_MIXEDMAP | VM_SEALED_SYSMAP, 132 &vdso_vvar_mapping); 133 } 134