1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/gfp.h> 4 #include <linux/init.h> 5 #include <linux/mm.h> 6 #include <linux/time_namespace.h> 7 #include <linux/types.h> 8 #include <linux/vdso_datastore.h> 9 #include <vdso/datapage.h> 10 11 static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {}; 12 13 #ifdef CONFIG_GENERIC_GETTIMEOFDAY 14 struct vdso_time_data *vdso_k_time_data __refdata = 15 (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE]; 16 17 static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE); 18 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */ 19 20 #ifdef CONFIG_VDSO_GETRANDOM 21 struct vdso_rng_data *vdso_k_rng_data __refdata = 22 (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE]; 23 24 static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE); 25 #endif /* CONFIG_VDSO_GETRANDOM */ 26 27 #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA 28 struct vdso_arch_data *vdso_k_arch_data __refdata = 29 (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE]; 30 #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */ 31 32 void __init vdso_setup_data_pages(void) 33 { 34 unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE); 35 struct page *pages; 36 37 /* 38 * Allocate the data pages dynamically. SPARC does not support mapping 39 * static pages to be mapped into userspace. 40 * It is also a requirement for mlockall() support. 41 * 42 * Do not use folios. In time namespaces the pages are mapped in a different order 43 * to userspace, which is not handled by the folio optimizations in finish_fault(). 44 */ 45 pages = alloc_pages(GFP_KERNEL, order); 46 if (!pages) 47 panic("Unable to allocate VDSO storage pages"); 48 49 /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */ 50 split_page(pages, order); 51 52 /* Move the data already written by other subsystems to the new pages */ 53 memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE); 54 55 if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) 56 vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET); 57 58 if (IS_ENABLED(CONFIG_VDSO_GETRANDOM)) 59 vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET); 60 61 if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) 62 vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START); 63 } 64 65 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, 66 struct vm_area_struct *vma, struct vm_fault *vmf) 67 { 68 struct page *page, *timens_page; 69 70 timens_page = find_timens_vvar_page(vma); 71 72 switch (vmf->pgoff) { 73 case VDSO_TIME_PAGE_OFFSET: 74 if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) 75 return VM_FAULT_SIGBUS; 76 page = virt_to_page(vdso_k_time_data); 77 if (timens_page) { 78 /* 79 * Fault in VVAR page too, since it will be accessed 80 * to get clock data anyway. 81 */ 82 unsigned long addr; 83 vm_fault_t err; 84 85 addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; 86 err = vmf_insert_page(vma, addr, page); 87 if (unlikely(err & VM_FAULT_ERROR)) 88 return err; 89 page = timens_page; 90 } 91 break; 92 case VDSO_TIMENS_PAGE_OFFSET: 93 /* 94 * If a task belongs to a time namespace then a namespace 95 * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and 96 * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET 97 * offset. 98 * See also the comment near timens_setup_vdso_data(). 99 */ 100 if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page) 101 return VM_FAULT_SIGBUS; 102 page = virt_to_page(vdso_k_time_data); 103 break; 104 case VDSO_RNG_PAGE_OFFSET: 105 if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM)) 106 return VM_FAULT_SIGBUS; 107 page = virt_to_page(vdso_k_rng_data); 108 break; 109 case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END: 110 if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) 111 return VM_FAULT_SIGBUS; 112 page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START; 113 break; 114 default: 115 return VM_FAULT_SIGBUS; 116 } 117 118 get_page(page); 119 vmf->page = page; 120 return 0; 121 } 122 123 const struct vm_special_mapping vdso_vvar_mapping = { 124 .name = "[vvar]", 125 .fault = vvar_fault, 126 }; 127 128 struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr) 129 { 130 return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, 131 VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | 132 VM_MIXEDMAP | VM_SEALED_SYSMAP, 133 &vdso_vvar_mapping); 134 } 135 136 #ifdef CONFIG_TIME_NS 137 /* 138 * The vvar page layout depends on whether a task belongs to the root or 139 * non-root time namespace. Whenever a task changes its namespace, the VVAR 140 * page tables are cleared and then they will be re-faulted with a 141 * corresponding layout. 142 * See also the comment near timens_setup_vdso_clock_data() for details. 143 */ 144 int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) 145 { 146 struct mm_struct *mm = task->mm; 147 struct vm_area_struct *vma; 148 VMA_ITERATOR(vmi, mm, 0); 149 150 mmap_read_lock(mm); 151 for_each_vma(vmi, vma) { 152 if (vma_is_special_mapping(vma, &vdso_vvar_mapping)) 153 zap_vma_pages(vma); 154 } 155 mmap_read_unlock(mm); 156 157 return 0; 158 } 159 #endif 160