1 /* 2 * VDSO implementations. 3 * 4 * Copyright (C) 2012 ARM Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * Author: Will Deacon <will.deacon@arm.com> 19 */ 20 21 #include <linux/cache.h> 22 #include <linux/clocksource.h> 23 #include <linux/elf.h> 24 #include <linux/err.h> 25 #include <linux/errno.h> 26 #include <linux/gfp.h> 27 #include <linux/kernel.h> 28 #include <linux/mm.h> 29 #include <linux/sched.h> 30 #include <linux/signal.h> 31 #include <linux/slab.h> 32 #include <linux/timekeeper_internal.h> 33 #include <linux/vmalloc.h> 34 35 #include <asm/cacheflush.h> 36 #include <asm/signal32.h> 37 #include <asm/vdso.h> 38 #include <asm/vdso_datapage.h> 39 40 extern char vdso_start[], vdso_end[]; 41 static unsigned long vdso_pages __ro_after_init; 42 43 /* 44 * The vDSO data page. 45 */ 46 static union { 47 struct vdso_data data; 48 u8 page[PAGE_SIZE]; 49 } vdso_data_store __page_aligned_data; 50 struct vdso_data *vdso_data = &vdso_data_store.data; 51 52 #ifdef CONFIG_COMPAT 53 /* 54 * Create and map the vectors page for AArch32 tasks. 55 */ 56 #define C_VECTORS 0 57 #define C_SIGPAGE 1 58 #define C_PAGES (C_SIGPAGE + 1) 59 static struct page *aarch32_vdso_pages[C_PAGES] __ro_after_init; 60 static const struct vm_special_mapping aarch32_vdso_spec[C_PAGES] = { 61 { 62 .name = "[vectors]", /* ABI */ 63 .pages = &aarch32_vdso_pages[C_VECTORS], 64 }, 65 { 66 .name = "[sigpage]", /* ABI */ 67 .pages = &aarch32_vdso_pages[C_SIGPAGE], 68 }, 69 }; 70 71 static int aarch32_alloc_kuser_vdso_page(void) 72 { 73 extern char __kuser_helper_start[], __kuser_helper_end[]; 74 int kuser_sz = __kuser_helper_end - __kuser_helper_start; 75 unsigned long vdso_page; 76 77 if (!IS_ENABLED(CONFIG_KUSER_HELPERS)) 78 return 0; 79 80 vdso_page = get_zeroed_page(GFP_ATOMIC); 81 if (!vdso_page) 82 return -ENOMEM; 83 84 memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start, 85 kuser_sz); 86 aarch32_vdso_pages[C_VECTORS] = virt_to_page(vdso_page); 87 flush_dcache_page(aarch32_vdso_pages[C_VECTORS]); 88 return 0; 89 } 90 91 static int __init aarch32_alloc_vdso_pages(void) 92 { 93 extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[]; 94 int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start; 95 unsigned long sigpage; 96 int ret; 97 98 sigpage = get_zeroed_page(GFP_ATOMIC); 99 if (!sigpage) 100 return -ENOMEM; 101 102 memcpy((void *)sigpage, __aarch32_sigret_code_start, sigret_sz); 103 aarch32_vdso_pages[C_SIGPAGE] = virt_to_page(sigpage); 104 flush_dcache_page(aarch32_vdso_pages[C_SIGPAGE]); 105 106 ret = aarch32_alloc_kuser_vdso_page(); 107 if (ret) 108 free_page(sigpage); 109 110 return ret; 111 } 112 arch_initcall(aarch32_alloc_vdso_pages); 113 114 static int aarch32_kuser_helpers_setup(struct mm_struct *mm) 115 { 116 void *ret; 117 118 if (!IS_ENABLED(CONFIG_KUSER_HELPERS)) 119 return 0; 120 121 /* 122 * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's 123 * not safe to CoW the page containing the CPU exception vectors. 124 */ 125 ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE, 126 VM_READ | VM_EXEC | 127 VM_MAYREAD | VM_MAYEXEC, 128 &aarch32_vdso_spec[C_VECTORS]); 129 130 return PTR_ERR_OR_ZERO(ret); 131 } 132 133 static int aarch32_sigreturn_setup(struct mm_struct *mm) 134 { 135 unsigned long addr; 136 void *ret; 137 138 addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0); 139 if (IS_ERR_VALUE(addr)) { 140 ret = ERR_PTR(addr); 141 goto out; 142 } 143 144 /* 145 * VM_MAYWRITE is required to allow gdb to Copy-on-Write and 146 * set breakpoints. 147 */ 148 ret = _install_special_mapping(mm, addr, PAGE_SIZE, 149 VM_READ | VM_EXEC | VM_MAYREAD | 150 VM_MAYWRITE | VM_MAYEXEC, 151 &aarch32_vdso_spec[C_SIGPAGE]); 152 if (IS_ERR(ret)) 153 goto out; 154 155 mm->context.vdso = (void *)addr; 156 157 out: 158 return PTR_ERR_OR_ZERO(ret); 159 } 160 161 int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) 162 { 163 struct mm_struct *mm = current->mm; 164 int ret; 165 166 if (down_write_killable(&mm->mmap_sem)) 167 return -EINTR; 168 169 ret = aarch32_kuser_helpers_setup(mm); 170 if (ret) 171 goto out; 172 173 ret = aarch32_sigreturn_setup(mm); 174 175 out: 176 up_write(&mm->mmap_sem); 177 return ret; 178 } 179 #endif /* CONFIG_COMPAT */ 180 181 static int vdso_mremap(const struct vm_special_mapping *sm, 182 struct vm_area_struct *new_vma) 183 { 184 unsigned long new_size = new_vma->vm_end - new_vma->vm_start; 185 unsigned long vdso_size = vdso_end - vdso_start; 186 187 if (vdso_size != new_size) 188 return -EINVAL; 189 190 current->mm->context.vdso = (void *)new_vma->vm_start; 191 192 return 0; 193 } 194 195 static struct vm_special_mapping vdso_spec[2] __ro_after_init = { 196 { 197 .name = "[vvar]", 198 }, 199 { 200 .name = "[vdso]", 201 .mremap = vdso_mremap, 202 }, 203 }; 204 205 static int __init vdso_init(void) 206 { 207 int i; 208 struct page **vdso_pagelist; 209 unsigned long pfn; 210 211 if (memcmp(vdso_start, "\177ELF", 4)) { 212 pr_err("vDSO is not a valid ELF object!\n"); 213 return -EINVAL; 214 } 215 216 vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; 217 218 /* Allocate the vDSO pagelist, plus a page for the data. */ 219 vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), 220 GFP_KERNEL); 221 if (vdso_pagelist == NULL) 222 return -ENOMEM; 223 224 /* Grab the vDSO data page. */ 225 vdso_pagelist[0] = phys_to_page(__pa_symbol(vdso_data)); 226 227 228 /* Grab the vDSO code pages. */ 229 pfn = sym_to_pfn(vdso_start); 230 231 for (i = 0; i < vdso_pages; i++) 232 vdso_pagelist[i + 1] = pfn_to_page(pfn + i); 233 234 vdso_spec[0].pages = &vdso_pagelist[0]; 235 vdso_spec[1].pages = &vdso_pagelist[1]; 236 237 return 0; 238 } 239 arch_initcall(vdso_init); 240 241 int arch_setup_additional_pages(struct linux_binprm *bprm, 242 int uses_interp) 243 { 244 struct mm_struct *mm = current->mm; 245 unsigned long vdso_base, vdso_text_len, vdso_mapping_len; 246 void *ret; 247 248 vdso_text_len = vdso_pages << PAGE_SHIFT; 249 /* Be sure to map the data page */ 250 vdso_mapping_len = vdso_text_len + PAGE_SIZE; 251 252 if (down_write_killable(&mm->mmap_sem)) 253 return -EINTR; 254 vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); 255 if (IS_ERR_VALUE(vdso_base)) { 256 ret = ERR_PTR(vdso_base); 257 goto up_fail; 258 } 259 ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, 260 VM_READ|VM_MAYREAD, 261 &vdso_spec[0]); 262 if (IS_ERR(ret)) 263 goto up_fail; 264 265 vdso_base += PAGE_SIZE; 266 mm->context.vdso = (void *)vdso_base; 267 ret = _install_special_mapping(mm, vdso_base, vdso_text_len, 268 VM_READ|VM_EXEC| 269 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, 270 &vdso_spec[1]); 271 if (IS_ERR(ret)) 272 goto up_fail; 273 274 275 up_write(&mm->mmap_sem); 276 return 0; 277 278 up_fail: 279 mm->context.vdso = NULL; 280 up_write(&mm->mmap_sem); 281 return PTR_ERR(ret); 282 } 283 284 /* 285 * Update the vDSO data page to keep in sync with kernel timekeeping. 286 */ 287 void update_vsyscall(struct timekeeper *tk) 288 { 289 u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct; 290 291 ++vdso_data->tb_seq_count; 292 smp_wmb(); 293 294 vdso_data->use_syscall = use_syscall; 295 vdso_data->xtime_coarse_sec = tk->xtime_sec; 296 vdso_data->xtime_coarse_nsec = tk->tkr_mono.xtime_nsec >> 297 tk->tkr_mono.shift; 298 vdso_data->wtm_clock_sec = tk->wall_to_monotonic.tv_sec; 299 vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; 300 301 /* Read without the seqlock held by clock_getres() */ 302 WRITE_ONCE(vdso_data->hrtimer_res, hrtimer_resolution); 303 304 if (!use_syscall) { 305 /* tkr_mono.cycle_last == tkr_raw.cycle_last */ 306 vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; 307 vdso_data->raw_time_sec = tk->raw_sec; 308 vdso_data->raw_time_nsec = tk->tkr_raw.xtime_nsec; 309 vdso_data->xtime_clock_sec = tk->xtime_sec; 310 vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec; 311 vdso_data->cs_mono_mult = tk->tkr_mono.mult; 312 vdso_data->cs_raw_mult = tk->tkr_raw.mult; 313 /* tkr_mono.shift == tkr_raw.shift */ 314 vdso_data->cs_shift = tk->tkr_mono.shift; 315 } 316 317 smp_wmb(); 318 ++vdso_data->tb_seq_count; 319 } 320 321 void update_vsyscall_tz(void) 322 { 323 vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; 324 vdso_data->tz_dsttime = sys_tz.tz_dsttime; 325 } 326