1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
4 */
5
6 #include <linux/mm.h>
7 #include <linux/module.h>
8 #include <linux/sched/signal.h>
9
10 #include <asm/tlbflush.h>
11 #include <asm/mmu_context.h>
12 #include <as-layout.h>
13 #include <mem_user.h>
14 #include <os.h>
15 #include <skas.h>
16 #include <kern_util.h>
17
18 struct vm_ops {
19 struct mm_id *mm_idp;
20
21 int (*mmap)(struct mm_id *mm_idp,
22 unsigned long virt, unsigned long len, int prot,
23 int phys_fd, unsigned long long offset);
24 int (*unmap)(struct mm_id *mm_idp,
25 unsigned long virt, unsigned long len);
26 int (*mprotect)(struct mm_id *mm_idp,
27 unsigned long virt, unsigned long len,
28 unsigned int prot);
29 };
30
kern_map(struct mm_id * mm_idp,unsigned long virt,unsigned long len,int prot,int phys_fd,unsigned long long offset)31 static int kern_map(struct mm_id *mm_idp,
32 unsigned long virt, unsigned long len, int prot,
33 int phys_fd, unsigned long long offset)
34 {
35 /* TODO: Why is executable needed to be always set in the kernel? */
36 return os_map_memory((void *)virt, phys_fd, offset, len,
37 prot & UM_PROT_READ, prot & UM_PROT_WRITE,
38 1);
39 }
40
kern_unmap(struct mm_id * mm_idp,unsigned long virt,unsigned long len)41 static int kern_unmap(struct mm_id *mm_idp,
42 unsigned long virt, unsigned long len)
43 {
44 return os_unmap_memory((void *)virt, len);
45 }
46
kern_mprotect(struct mm_id * mm_idp,unsigned long virt,unsigned long len,unsigned int prot)47 static int kern_mprotect(struct mm_id *mm_idp,
48 unsigned long virt, unsigned long len,
49 unsigned int prot)
50 {
51 return os_protect_memory((void *)virt, len,
52 prot & UM_PROT_READ, prot & UM_PROT_WRITE,
53 1);
54 }
55
report_enomem(void)56 void report_enomem(void)
57 {
58 printk(KERN_ERR "UML ran out of memory on the host side! "
59 "This can happen due to a memory limitation or "
60 "vm.max_map_count has been reached.\n");
61 }
62
update_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct vm_ops * ops)63 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
64 unsigned long end,
65 struct vm_ops *ops)
66 {
67 pte_t *pte;
68 int r, w, x, prot, ret = 0;
69
70 pte = pte_offset_kernel(pmd, addr);
71 do {
72 r = pte_read(*pte);
73 w = pte_write(*pte);
74 x = pte_exec(*pte);
75 if (!pte_young(*pte)) {
76 r = 0;
77 w = 0;
78 } else if (!pte_dirty(*pte))
79 w = 0;
80
81 prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
82 (x ? UM_PROT_EXEC : 0));
83 if (pte_newpage(*pte)) {
84 if (pte_present(*pte)) {
85 if (pte_newpage(*pte)) {
86 __u64 offset;
87 unsigned long phys =
88 pte_val(*pte) & PAGE_MASK;
89 int fd = phys_mapping(phys, &offset);
90
91 ret = ops->mmap(ops->mm_idp, addr,
92 PAGE_SIZE, prot, fd,
93 offset);
94 }
95 } else
96 ret = ops->unmap(ops->mm_idp, addr, PAGE_SIZE);
97 } else if (pte_newprot(*pte))
98 ret = ops->mprotect(ops->mm_idp, addr, PAGE_SIZE, prot);
99 *pte = pte_mkuptodate(*pte);
100 } while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
101 return ret;
102 }
103
update_pmd_range(pud_t * pud,unsigned long addr,unsigned long end,struct vm_ops * ops)104 static inline int update_pmd_range(pud_t *pud, unsigned long addr,
105 unsigned long end,
106 struct vm_ops *ops)
107 {
108 pmd_t *pmd;
109 unsigned long next;
110 int ret = 0;
111
112 pmd = pmd_offset(pud, addr);
113 do {
114 next = pmd_addr_end(addr, end);
115 if (!pmd_present(*pmd)) {
116 if (pmd_newpage(*pmd)) {
117 ret = ops->unmap(ops->mm_idp, addr,
118 next - addr);
119 pmd_mkuptodate(*pmd);
120 }
121 }
122 else ret = update_pte_range(pmd, addr, next, ops);
123 } while (pmd++, addr = next, ((addr < end) && !ret));
124 return ret;
125 }
126
update_pud_range(p4d_t * p4d,unsigned long addr,unsigned long end,struct vm_ops * ops)127 static inline int update_pud_range(p4d_t *p4d, unsigned long addr,
128 unsigned long end,
129 struct vm_ops *ops)
130 {
131 pud_t *pud;
132 unsigned long next;
133 int ret = 0;
134
135 pud = pud_offset(p4d, addr);
136 do {
137 next = pud_addr_end(addr, end);
138 if (!pud_present(*pud)) {
139 if (pud_newpage(*pud)) {
140 ret = ops->unmap(ops->mm_idp, addr,
141 next - addr);
142 pud_mkuptodate(*pud);
143 }
144 }
145 else ret = update_pmd_range(pud, addr, next, ops);
146 } while (pud++, addr = next, ((addr < end) && !ret));
147 return ret;
148 }
149
update_p4d_range(pgd_t * pgd,unsigned long addr,unsigned long end,struct vm_ops * ops)150 static inline int update_p4d_range(pgd_t *pgd, unsigned long addr,
151 unsigned long end,
152 struct vm_ops *ops)
153 {
154 p4d_t *p4d;
155 unsigned long next;
156 int ret = 0;
157
158 p4d = p4d_offset(pgd, addr);
159 do {
160 next = p4d_addr_end(addr, end);
161 if (!p4d_present(*p4d)) {
162 if (p4d_newpage(*p4d)) {
163 ret = ops->unmap(ops->mm_idp, addr,
164 next - addr);
165 p4d_mkuptodate(*p4d);
166 }
167 } else
168 ret = update_pud_range(p4d, addr, next, ops);
169 } while (p4d++, addr = next, ((addr < end) && !ret));
170 return ret;
171 }
172
um_tlb_sync(struct mm_struct * mm)173 int um_tlb_sync(struct mm_struct *mm)
174 {
175 pgd_t *pgd;
176 struct vm_ops ops;
177 unsigned long addr = mm->context.sync_tlb_range_from, next;
178 int ret = 0;
179
180 if (mm->context.sync_tlb_range_to == 0)
181 return 0;
182
183 ops.mm_idp = &mm->context.id;
184 if (mm == &init_mm) {
185 ops.mmap = kern_map;
186 ops.unmap = kern_unmap;
187 ops.mprotect = kern_mprotect;
188 } else {
189 ops.mmap = map;
190 ops.unmap = unmap;
191 ops.mprotect = protect;
192 }
193
194 pgd = pgd_offset(mm, addr);
195 do {
196 next = pgd_addr_end(addr, mm->context.sync_tlb_range_to);
197 if (!pgd_present(*pgd)) {
198 if (pgd_newpage(*pgd)) {
199 ret = ops.unmap(ops.mm_idp, addr,
200 next - addr);
201 pgd_mkuptodate(*pgd);
202 }
203 } else
204 ret = update_p4d_range(pgd, addr, next, &ops);
205 } while (pgd++, addr = next,
206 ((addr < mm->context.sync_tlb_range_to) && !ret));
207
208 if (ret == -ENOMEM)
209 report_enomem();
210
211 mm->context.sync_tlb_range_from = 0;
212 mm->context.sync_tlb_range_to = 0;
213
214 return ret;
215 }
216
flush_tlb_all(void)217 void flush_tlb_all(void)
218 {
219 /*
220 * Don't bother flushing if this address space is about to be
221 * destroyed.
222 */
223 if (atomic_read(¤t->mm->mm_users) == 0)
224 return;
225
226 flush_tlb_mm(current->mm);
227 }
228
flush_tlb_mm(struct mm_struct * mm)229 void flush_tlb_mm(struct mm_struct *mm)
230 {
231 struct vm_area_struct *vma;
232 VMA_ITERATOR(vmi, mm, 0);
233
234 for_each_vma(vmi, vma)
235 um_tlb_mark_sync(mm, vma->vm_start, vma->vm_end);
236 }
237