1 /*-
2 * Copyright (c) 2004 Tim J. Robbins
3 * Copyright (c) 2002 Doug Rabson
4 * Copyright (c) 2000 Marcel Moolenaar
5 * Copyright (c) 1994-1995 Søren Schmidt
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer
13 * in this position and unchanged.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. The name of the author may not be used to endorse or promote products
18 * derived from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include <sys/fcntl.h>
33 #include <sys/file.h>
34 #include <sys/ktr.h>
35 #include <sys/lock.h>
36 #include <sys/malloc.h>
37 #include <sys/mman.h>
38 #include <sys/proc.h>
39 #include <sys/resourcevar.h>
40 #include <sys/rwlock.h>
41 #include <sys/syscallsubr.h>
42 #include <sys/sysent.h>
43 #include <sys/sysproto.h>
44
45 #include <vm/pmap.h>
46 #include <vm/vm_extern.h>
47 #include <vm/vm_map.h>
48 #include <vm/vm_object.h>
49
50 #include <compat/linux/linux_emul.h>
51 #include <compat/linux/linux_mmap.h>
52 #include <compat/linux/linux_persona.h>
53 #include <compat/linux/linux_util.h>
54
55 #define STACK_SIZE (2 * 1024 * 1024)
56 #define GUARD_SIZE (4 * PAGE_SIZE)
57
58 #if defined(__amd64__)
59 static void linux_fixup_prot(struct thread *td, int *prot);
60 #endif
61
62 static int
linux_mmap_check_fp(struct file * fp,int flags,int prot,int maxprot)63 linux_mmap_check_fp(struct file *fp, int flags, int prot, int maxprot)
64 {
65
66 /* Linux returns EBADF if mmap() is called on an O_PATH file descriptor */
67 if (fp->f_ops == &path_fileops)
68 return (EBADF);
69
70 /* Linux mmap() just fails for O_WRONLY files */
71 if ((fp->f_flag & FREAD) == 0)
72 return (EACCES);
73
74 return (0);
75 }
76
77 int
linux_mmap_common(struct thread * td,uintptr_t addr,size_t len,int prot,int flags,int fd,off_t pos)78 linux_mmap_common(struct thread *td, uintptr_t addr, size_t len, int prot,
79 int flags, int fd, off_t pos)
80 {
81 struct mmap_req mr, mr_fixed;
82 struct proc *p = td->td_proc;
83 struct vmspace *vms = td->td_proc->p_vmspace;
84 int bsd_flags, error;
85
86 LINUX_CTR6(mmap2, "0x%lx, %ld, %ld, 0x%08lx, %ld, 0x%lx",
87 addr, len, prot, flags, fd, pos);
88
89 error = 0;
90 bsd_flags = 0;
91
92 /*
93 * Linux mmap(2):
94 * You must specify exactly one of MAP_SHARED and MAP_PRIVATE
95 */
96 if (!((flags & LINUX_MAP_SHARED) ^ (flags & LINUX_MAP_PRIVATE)))
97 return (EINVAL);
98
99 if (flags & LINUX_MAP_SHARED)
100 bsd_flags |= MAP_SHARED;
101 if (flags & LINUX_MAP_PRIVATE)
102 bsd_flags |= MAP_PRIVATE;
103 if (flags & LINUX_MAP_FIXED)
104 bsd_flags |= MAP_FIXED;
105 if (flags & LINUX_MAP_ANON) {
106 /* Enforce pos to be on page boundary, then ignore. */
107 if ((pos & PAGE_MASK) != 0)
108 return (EINVAL);
109 pos = 0;
110 bsd_flags |= MAP_ANON;
111 } else
112 bsd_flags |= MAP_NOSYNC;
113 if (flags & LINUX_MAP_GROWSDOWN)
114 bsd_flags |= MAP_STACK;
115
116 #if defined(__amd64__)
117 /*
118 * According to the Linux mmap(2) man page, "MAP_32BIT flag
119 * is ignored when MAP_FIXED is set."
120 */
121 if ((flags & LINUX_MAP_32BIT) && (flags & LINUX_MAP_FIXED) == 0)
122 bsd_flags |= MAP_32BIT;
123
124 /*
125 * PROT_READ, PROT_WRITE, or PROT_EXEC implies PROT_READ and PROT_EXEC
126 * on Linux/i386 if the binary requires executable stack.
127 * We do this only for IA32 emulation as on native i386 this is does not
128 * make sense without PAE.
129 *
130 * XXX. Linux checks that the file system is not mounted with noexec.
131 */
132 linux_fixup_prot(td, &prot);
133 #endif
134
135 /* Linux does not check file descriptor when MAP_ANONYMOUS is set. */
136 fd = (bsd_flags & MAP_ANON) ? -1 : fd;
137 if (flags & LINUX_MAP_GROWSDOWN) {
138 /*
139 * The Linux MAP_GROWSDOWN option does not limit auto
140 * growth of the region. Linux mmap with this option
141 * takes as addr the initial BOS, and as len, the initial
142 * region size. It can then grow down from addr without
143 * limit. However, Linux threads has an implicit internal
144 * limit to stack size of STACK_SIZE. Its just not
145 * enforced explicitly in Linux. But, here we impose
146 * a limit of (STACK_SIZE - GUARD_SIZE) on the stack
147 * region, since we can do this with our mmap.
148 *
149 * Our mmap with MAP_STACK takes addr as the maximum
150 * downsize limit on BOS, and as len the max size of
151 * the region. It then maps the top SGROWSIZ bytes,
152 * and auto grows the region down, up to the limit
153 * in addr.
154 *
155 * If we don't use the MAP_STACK option, the effect
156 * of this code is to allocate a stack region of a
157 * fixed size of (STACK_SIZE - GUARD_SIZE).
158 */
159
160 if ((caddr_t)addr + len > vms->vm_maxsaddr) {
161 /*
162 * Some Linux apps will attempt to mmap
163 * thread stacks near the top of their
164 * address space. If their TOS is greater
165 * than vm_maxsaddr, vm_map_growstack()
166 * will confuse the thread stack with the
167 * process stack and deliver a SEGV if they
168 * attempt to grow the thread stack past their
169 * current stacksize rlimit. To avoid this,
170 * adjust vm_maxsaddr upwards to reflect
171 * the current stacksize rlimit rather
172 * than the maximum possible stacksize.
173 * It would be better to adjust the
174 * mmap'ed region, but some apps do not check
175 * mmap's return value.
176 */
177 PROC_LOCK(p);
178 vms->vm_maxsaddr = (char *)round_page(vms->vm_stacktop) -
179 lim_cur_proc(p, RLIMIT_STACK);
180 PROC_UNLOCK(p);
181 }
182
183 /*
184 * This gives us our maximum stack size and a new BOS.
185 * If we're using VM_STACK, then mmap will just map
186 * the top SGROWSIZ bytes, and let the stack grow down
187 * to the limit at BOS. If we're not using VM_STACK
188 * we map the full stack, since we don't have a way
189 * to autogrow it.
190 */
191 if (len <= STACK_SIZE - GUARD_SIZE) {
192 addr = addr - (STACK_SIZE - GUARD_SIZE - len);
193 len = STACK_SIZE - GUARD_SIZE;
194 }
195 }
196
197 /*
198 * FreeBSD is free to ignore the address hint if MAP_FIXED wasn't
199 * passed. However, some Linux applications, like the ART runtime,
200 * depend on the hint. If the MAP_FIXED wasn't passed, but the
201 * address is not zero, try with MAP_FIXED and MAP_EXCL first,
202 * and fall back to the normal behaviour if that fails.
203 */
204 mr = (struct mmap_req) {
205 .mr_hint = addr,
206 .mr_len = len,
207 .mr_prot = prot,
208 .mr_flags = bsd_flags,
209 .mr_fd = fd,
210 .mr_pos = pos,
211 .mr_check_fp_fn = linux_mmap_check_fp,
212 };
213 if (addr != 0 && (bsd_flags & MAP_FIXED) == 0 &&
214 (bsd_flags & MAP_EXCL) == 0) {
215 mr_fixed = mr;
216 mr_fixed.mr_flags |= MAP_FIXED | MAP_EXCL;
217 error = kern_mmap(td, &mr_fixed);
218 if (error == 0)
219 goto out;
220 }
221
222 error = kern_mmap(td, &mr);
223 out:
224 LINUX_CTR2(mmap2, "return: %d (%p)", error, td->td_retval[0]);
225
226 return (error);
227 }
228
229 int
linux_mprotect_common(struct thread * td,uintptr_t addr,size_t len,int prot)230 linux_mprotect_common(struct thread *td, uintptr_t addr, size_t len, int prot)
231 {
232 int flags = 0;
233
234 /* XXX Ignore PROT_GROWSUP for now. */
235 prot &= ~LINUX_PROT_GROWSUP;
236 if ((prot & ~(LINUX_PROT_GROWSDOWN | PROT_READ | PROT_WRITE |
237 PROT_EXEC)) != 0)
238 return (EINVAL);
239 if ((prot & LINUX_PROT_GROWSDOWN) != 0) {
240 prot &= ~LINUX_PROT_GROWSDOWN;
241 flags |= VM_MAP_PROTECT_GROWSDOWN;
242 }
243
244 #if defined(__amd64__)
245 linux_fixup_prot(td, &prot);
246 #endif
247 return (kern_mprotect(td, addr, len, prot, flags));
248 }
249
250 /*
251 * x86 memory protection keys. The common entry points perform the
252 * parameter validation Linux applies regardless of hardware support,
253 * then defer to the machine-dependent back end.
254 */
255
256 int
linux_pkey_alloc_common(struct thread * td,uint64_t flags,uint64_t init_val)257 linux_pkey_alloc_common(struct thread *td, uint64_t flags, uint64_t init_val)
258 {
259
260 if (flags != 0)
261 return (EINVAL);
262 if ((init_val & ~(uint64_t)LINUX_PKEY_ACCESS_MASK) != 0)
263 return (EINVAL);
264 return (linux_pkey_alloc_machdep(td, init_val));
265 }
266
267 int
linux_pkey_free_common(struct thread * td,int pkey)268 linux_pkey_free_common(struct thread *td, int pkey)
269 {
270
271 if (pkey < 0 || pkey >= LINUX_PKEY_MAX)
272 return (EINVAL);
273 return (linux_pkey_free_machdep(td, pkey));
274 }
275
276 int
linux_pkey_mprotect_common(struct thread * td,uintptr_t addr,size_t len,int prot,int pkey)277 linux_pkey_mprotect_common(struct thread *td, uintptr_t addr, size_t len,
278 int prot, int pkey)
279 {
280
281 if (pkey < -1 || pkey >= LINUX_PKEY_MAX)
282 return (EINVAL);
283 if (pkey == -1)
284 return (linux_mprotect_common(td, addr, len, prot));
285 return (linux_pkey_mprotect_machdep(td, addr, len, prot, pkey));
286 }
287
288 /*
289 * Implement Linux madvise(MADV_DONTNEED), which has unusual semantics: for
290 * anonymous memory, pages in the range are immediately discarded.
291 */
292 static int
linux_madvise_dontneed(struct thread * td,vm_offset_t start,vm_offset_t end)293 linux_madvise_dontneed(struct thread *td, vm_offset_t start, vm_offset_t end)
294 {
295 vm_map_t map;
296 vm_map_entry_t entry;
297 vm_object_t backing_object, object;
298 vm_offset_t estart, eend;
299 vm_pindex_t pstart, pend;
300 int error;
301
302 map = &td->td_proc->p_vmspace->vm_map;
303
304 if (!vm_map_range_valid(map, start, end))
305 return (EINVAL);
306 start = trunc_page(start);
307 end = round_page(end);
308
309 error = 0;
310 vm_map_lock_read(map);
311 if (!vm_map_lookup_entry(map, start, &entry))
312 entry = vm_map_entry_succ(entry);
313 for (; entry->start < end; entry = vm_map_entry_succ(entry)) {
314 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
315 continue;
316
317 if (entry->wired_count != 0) {
318 error = EINVAL;
319 break;
320 }
321
322 object = entry->object.vm_object;
323 if (object == NULL)
324 continue;
325 if ((object->flags & (OBJ_UNMANAGED | OBJ_FICTITIOUS)) != 0)
326 continue;
327
328 pstart = OFF_TO_IDX(entry->offset);
329 if (start > entry->start) {
330 pstart += atop(start - entry->start);
331 estart = start;
332 } else {
333 estart = entry->start;
334 }
335 pend = OFF_TO_IDX(entry->offset) +
336 atop(entry->end - entry->start);
337 if (entry->end > end) {
338 pend -= atop(entry->end - end);
339 eend = end;
340 } else {
341 eend = entry->end;
342 }
343
344 if ((object->flags & (OBJ_ANON | OBJ_ONEMAPPING)) ==
345 (OBJ_ANON | OBJ_ONEMAPPING)) {
346 /*
347 * Singly-mapped anonymous memory is discarded. This
348 * does not match Linux's semantics when the object
349 * belongs to a shadow chain of length > 1, since
350 * subsequent faults may retrieve pages from an
351 * intermediate anonymous object. However, handling
352 * this case correctly introduces a fair bit of
353 * complexity.
354 */
355 VM_OBJECT_WLOCK(object);
356 if ((object->flags & OBJ_ONEMAPPING) != 0) {
357 vm_object_collapse(object);
358 vm_object_page_remove(object, pstart, pend, 0);
359 backing_object = object->backing_object;
360 if (backing_object != NULL &&
361 (backing_object->flags & OBJ_ANON) != 0)
362 linux_msg(td,
363 "possibly incorrect MADV_DONTNEED");
364 VM_OBJECT_WUNLOCK(object);
365 continue;
366 }
367 VM_OBJECT_WUNLOCK(object);
368 }
369
370 /*
371 * Handle shared mappings. Remove them outright instead of
372 * calling pmap_advise(), for consistency with Linux.
373 */
374 pmap_remove(map->pmap, estart, eend);
375 vm_object_madvise(object, pstart, pend, MADV_DONTNEED);
376 }
377 vm_map_unlock_read(map);
378
379 return (error);
380 }
381
382 int
linux_madvise_common(struct thread * td,uintptr_t addr,size_t len,int behav)383 linux_madvise_common(struct thread *td, uintptr_t addr, size_t len, int behav)
384 {
385
386 switch (behav) {
387 case LINUX_MADV_NORMAL:
388 return (kern_madvise(td, addr, len, MADV_NORMAL));
389 case LINUX_MADV_RANDOM:
390 return (kern_madvise(td, addr, len, MADV_RANDOM));
391 case LINUX_MADV_SEQUENTIAL:
392 return (kern_madvise(td, addr, len, MADV_SEQUENTIAL));
393 case LINUX_MADV_WILLNEED:
394 return (kern_madvise(td, addr, len, MADV_WILLNEED));
395 case LINUX_MADV_DONTNEED:
396 return (linux_madvise_dontneed(td, addr, addr + len));
397 case LINUX_MADV_FREE:
398 return (kern_madvise(td, addr, len, MADV_FREE));
399 case LINUX_MADV_REMOVE:
400 linux_msg(curthread, "unsupported madvise MADV_REMOVE");
401 return (EINVAL);
402 case LINUX_MADV_DONTFORK:
403 return (kern_minherit(td, addr, len, INHERIT_NONE));
404 case LINUX_MADV_DOFORK:
405 return (kern_minherit(td, addr, len, INHERIT_COPY));
406 case LINUX_MADV_MERGEABLE:
407 linux_msg(curthread, "unsupported madvise MADV_MERGEABLE");
408 return (EINVAL);
409 case LINUX_MADV_UNMERGEABLE:
410 /* We don't merge anyway. */
411 return (0);
412 case LINUX_MADV_HUGEPAGE:
413 /* Ignored; on FreeBSD huge pages are always on. */
414 return (0);
415 case LINUX_MADV_NOHUGEPAGE:
416 #if 0
417 /*
418 * Don't warn - Firefox uses it a lot, and in real Linux it's
419 * an optional feature.
420 */
421 linux_msg(curthread, "unsupported madvise MADV_NOHUGEPAGE");
422 #endif
423 return (EINVAL);
424 case LINUX_MADV_DONTDUMP:
425 return (kern_madvise(td, addr, len, MADV_NOCORE));
426 case LINUX_MADV_DODUMP:
427 return (kern_madvise(td, addr, len, MADV_CORE));
428 case LINUX_MADV_WIPEONFORK:
429 return (kern_minherit(td, addr, len, INHERIT_ZERO));
430 case LINUX_MADV_KEEPONFORK:
431 return (kern_minherit(td, addr, len, INHERIT_COPY));
432 case LINUX_MADV_HWPOISON:
433 linux_msg(curthread, "unsupported madvise MADV_HWPOISON");
434 return (EINVAL);
435 case LINUX_MADV_SOFT_OFFLINE:
436 linux_msg(curthread, "unsupported madvise MADV_SOFT_OFFLINE");
437 return (EINVAL);
438 case -1:
439 /*
440 * -1 is sometimes used as a dummy value to detect simplistic
441 * madvise(2) stub implementations. This safeguard is used by
442 * BoringSSL, for example, before assuming MADV_WIPEONFORK is
443 * safe to use. Don't produce an "unsupported" error message
444 * for this special dummy value, which is unlikely to be used
445 * by any new advisory behavior feature.
446 */
447 return (EINVAL);
448 default:
449 linux_msg(curthread, "unsupported madvise behav %d", behav);
450 return (EINVAL);
451 }
452 }
453
454 #if defined(__amd64__)
455 static void
linux_fixup_prot(struct thread * td,int * prot)456 linux_fixup_prot(struct thread *td, int *prot)
457 {
458 struct linux_pemuldata *pem;
459
460 if (SV_PROC_FLAG(td->td_proc, SV_ILP32) && *prot & PROT_READ) {
461 pem = pem_find(td->td_proc);
462 if (pem->persona & LINUX_READ_IMPLIES_EXEC)
463 *prot |= PROT_EXEC;
464 }
465
466 }
467 #endif
468