1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
4 * <benh@kernel.crashing.org>
5 * Copyright (C) 2012 ARM Limited
6 * Copyright (C) 2015 Regents of the University of California
7 */
8
9 #include <linux/elf.h>
10 #include <linux/mm.h>
11 #include <linux/slab.h>
12 #include <linux/binfmts.h>
13 #include <linux/err.h>
14 #include <asm/page.h>
15 #include <asm/vdso.h>
16 #include <linux/vdso_datastore.h>
17 #include <vdso/datapage.h>
18 #include <vdso/vsyscall.h>
19
20 #define VVAR_SIZE (VDSO_NR_PAGES << PAGE_SHIFT)
21
22 struct __vdso_info {
23 const char *name;
24 const char *vdso_code_start;
25 const char *vdso_code_end;
26 unsigned long vdso_pages;
27 /* Code Mapping */
28 struct vm_special_mapping *cm;
29 };
30
31 static struct __vdso_info vdso_info;
32 #ifdef CONFIG_COMPAT
33 static struct __vdso_info compat_vdso_info;
34 #endif
35
vdso_mremap(const struct vm_special_mapping * sm,struct vm_area_struct * new_vma)36 static int vdso_mremap(const struct vm_special_mapping *sm,
37 struct vm_area_struct *new_vma)
38 {
39 current->mm->context.vdso = (void *)new_vma->vm_start;
40
41 return 0;
42 }
43
__vdso_init(struct __vdso_info * vdso_info)44 static void __init __vdso_init(struct __vdso_info *vdso_info)
45 {
46 unsigned int i;
47 struct page **vdso_pagelist;
48 unsigned long pfn;
49
50 if (memcmp(vdso_info->vdso_code_start, "\177ELF", 4))
51 panic("vDSO is not a valid ELF object!\n");
52
53 vdso_info->vdso_pages = (
54 vdso_info->vdso_code_end -
55 vdso_info->vdso_code_start) >>
56 PAGE_SHIFT;
57
58 vdso_pagelist = kzalloc_objs(struct page *, vdso_info->vdso_pages);
59 if (vdso_pagelist == NULL)
60 panic("vDSO kcalloc failed!\n");
61
62 /* Grab the vDSO code pages. */
63 pfn = sym_to_pfn(vdso_info->vdso_code_start);
64
65 for (i = 0; i < vdso_info->vdso_pages; i++)
66 vdso_pagelist[i] = pfn_to_page(pfn + i);
67
68 vdso_info->cm->pages = vdso_pagelist;
69 }
70
71 static struct vm_special_mapping rv_vdso_map __ro_after_init = {
72 .name = "[vdso]",
73 .mremap = vdso_mremap,
74 };
75
76 static struct __vdso_info vdso_info __ro_after_init = {
77 .name = "vdso",
78 .vdso_code_start = vdso_start,
79 .vdso_code_end = vdso_end,
80 .cm = &rv_vdso_map,
81 };
82
83 #ifdef CONFIG_COMPAT
84 static struct vm_special_mapping rv_compat_vdso_map __ro_after_init = {
85 .name = "[vdso]",
86 .mremap = vdso_mremap,
87 };
88
89 static struct __vdso_info compat_vdso_info __ro_after_init = {
90 .name = "compat_vdso",
91 .vdso_code_start = compat_vdso_start,
92 .vdso_code_end = compat_vdso_end,
93 .cm = &rv_compat_vdso_map,
94 };
95 #endif
96
vdso_init(void)97 static int __init vdso_init(void)
98 {
99 /* Hart implements zimop, expose cfi compiled vdso */
100 if (IS_ENABLED(CONFIG_RISCV_USER_CFI) &&
101 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP)) {
102 vdso_info.vdso_code_start = vdso_cfi_start;
103 vdso_info.vdso_code_end = vdso_cfi_end;
104 }
105
106 __vdso_init(&vdso_info);
107 #ifdef CONFIG_COMPAT
108 __vdso_init(&compat_vdso_info);
109 #endif
110
111 return 0;
112 }
113 arch_initcall(vdso_init);
114
__setup_additional_pages(struct mm_struct * mm,struct linux_binprm * bprm,int uses_interp,struct __vdso_info * vdso_info)115 static int __setup_additional_pages(struct mm_struct *mm,
116 struct linux_binprm *bprm,
117 int uses_interp,
118 struct __vdso_info *vdso_info)
119 {
120 unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
121 void *ret;
122
123 BUILD_BUG_ON(VDSO_NR_PAGES != __VDSO_PAGES);
124
125 vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT;
126 /* Be sure to map the data page */
127 vdso_mapping_len = vdso_text_len + VVAR_SIZE;
128
129 vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
130 if (IS_ERR_VALUE(vdso_base)) {
131 ret = ERR_PTR(vdso_base);
132 goto up_fail;
133 }
134
135 ret = vdso_install_vvar_mapping(mm, vdso_base);
136 if (IS_ERR(ret))
137 goto up_fail;
138
139 vdso_base += VVAR_SIZE;
140 mm->context.vdso = (void *)vdso_base;
141
142 ret =
143 _install_special_mapping(mm, vdso_base, vdso_text_len,
144 (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC | VM_SEALED_SYSMAP),
145 vdso_info->cm);
146
147 if (IS_ERR(ret))
148 goto up_fail;
149
150 return 0;
151
152 up_fail:
153 mm->context.vdso = NULL;
154 return PTR_ERR(ret);
155 }
156
157 #ifdef CONFIG_COMPAT
compat_arch_setup_additional_pages(struct linux_binprm * bprm,int uses_interp)158 int compat_arch_setup_additional_pages(struct linux_binprm *bprm,
159 int uses_interp)
160 {
161 struct mm_struct *mm = current->mm;
162 int ret;
163
164 if (mmap_write_lock_killable(mm))
165 return -EINTR;
166
167 ret = __setup_additional_pages(mm, bprm, uses_interp,
168 &compat_vdso_info);
169 mmap_write_unlock(mm);
170
171 return ret;
172 }
173 #endif
174
arch_setup_additional_pages(struct linux_binprm * bprm,int uses_interp)175 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
176 {
177 struct mm_struct *mm = current->mm;
178 int ret;
179
180 if (mmap_write_lock_killable(mm))
181 return -EINTR;
182
183 ret = __setup_additional_pages(mm, bprm, uses_interp, &vdso_info);
184 mmap_write_unlock(mm);
185
186 return ret;
187 }
188