1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Functions provided for exec functionality which however are 5 * specifically VMA-only logic. 6 */ 7 8 /* 9 * To allow for userland testing we place internal dependencies in 10 * vma_internal.h and external VMA API declarations in vma.h. 11 */ 12 #include "vma_internal.h" 13 #include "vma.h" 14 15 /* 16 * Relocate a VMA downwards by shift bytes. There cannot be any VMAs between 17 * this VMA and its relocated range, which will now reside at [vma->vm_start - 18 * shift, vma->vm_end - shift). 19 * 20 * This function is almost certainly NOT what you want for anything other than 21 * early executable temporary stack relocation. 22 */ 23 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) 24 { 25 /* 26 * The process proceeds as follows: 27 * 28 * 1) Use shift to calculate the new vma endpoints. 29 * 2) Extend vma to cover both the old and new ranges. This ensures the 30 * arguments passed to subsequent functions are consistent. 31 * 3) Move vma's page tables to the new range. 32 * 4) Free up any cleared pgd range. 33 * 5) Shrink the vma to cover only the new range. 34 */ 35 36 struct mm_struct *mm = vma->vm_mm; 37 unsigned long old_start = vma->vm_start; 38 unsigned long old_end = vma->vm_end; 39 unsigned long length = old_end - old_start; 40 unsigned long new_start = old_start - shift; 41 unsigned long new_end = old_end - shift; 42 VMA_ITERATOR(vmi, mm, new_start); 43 VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS, 44 vma_start_pgoff(vma), vma_start_anon_pgoff(vma)); 45 struct vm_area_struct *next; 46 struct mmu_gather tlb; 47 PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); 48 49 BUG_ON(new_start > new_end); 50 51 /* 52 * ensure there are no vmas between where we want to go 53 * and where we are 54 */ 55 if (vma != vma_next(&vmi)) 56 return -EFAULT; 57 58 vma_iter_prev_range(&vmi); 59 /* 60 * cover the whole range: [new_start, old_end) 61 */ 62 vmg.target = vma; 63 if (vma_expand(&vmg)) 64 return -ENOMEM; 65 66 /* 67 * move the page tables downwards, on failure we rely on 68 * process cleanup to remove whatever mess we made. 69 */ 70 pmc.for_stack = true; 71 if (length != move_page_tables(&pmc)) 72 return -ENOMEM; 73 74 tlb_gather_mmu(&tlb, mm); 75 next = vma_next(&vmi); 76 if (new_end > old_start) { 77 /* 78 * when the old and new regions overlap clear from new_end. 79 */ 80 free_pgd_range(&tlb, new_end, old_end, new_end, 81 next ? next->vm_start : USER_PGTABLES_CEILING); 82 } else { 83 /* 84 * otherwise, clean from old_start; this is done to not touch 85 * the address space in [new_end, old_start) some architectures 86 * have constraints on va-space that make this illegal (IA64) - 87 * for the others its just a little faster. 88 */ 89 free_pgd_range(&tlb, old_start, old_end, new_end, 90 next ? next->vm_start : USER_PGTABLES_CEILING); 91 } 92 tlb_finish_mmu(&tlb); 93 94 vma_prev(&vmi); 95 /* Shrink the vma to just the new range */ 96 return vma_shrink(&vmi, vma, new_end); 97 } 98 99 /* 100 * Establish the stack VMA in an execve'd process, located temporarily at the 101 * maximum stack address provided by the architecture. 102 * 103 * We later relocate this downwards in relocate_vma_down(). 104 * 105 * This function is almost certainly NOT what you want for anything other than 106 * early executable initialisation. 107 * 108 * On success, returns 0 and sets *vmap to the stack VMA and *top_mem_p to the 109 * maximum addressable location in the stack (that is capable of storing a 110 * system word of data). 111 */ 112 int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, 113 unsigned long *top_mem_p) 114 { 115 vma_flags_t flags = VMA_STACK_INCOMPLETE_SETUP; 116 struct vm_area_struct *vma; 117 int err; 118 119 /* VMA_STACK_FLAGS and VMA_STACK_INCOMPLETE_SETUP must not overlap. */ 120 VM_WARN_ON_ONCE(vma_flags_test_any_mask(&flags, VMA_STACK_FLAGS)); 121 122 vma = vm_area_alloc(mm); 123 if (!vma) 124 return -ENOMEM; 125 126 if (mmap_write_lock_killable(mm)) { 127 err = -EINTR; 128 goto err_free; 129 } 130 131 /* 132 * Need to be called with mmap write lock 133 * held, to avoid race with ksmd. 134 */ 135 err = ksm_execve(mm); 136 if (err) 137 goto err_ksm; 138 139 vma_flags_set_mask(&flags, VMA_STACK_FLAGS); 140 vma_set_anonymous(vma); 141 142 /* 143 * Place the stack at the largest stack address the architecture 144 * supports. Later, we'll move this to an appropriate place. We don't 145 * use STACK_TOP because that can depend on attributes which aren't 146 * configured yet. 147 */ 148 vma->vm_end = STACK_TOP_MAX; 149 vma->vm_start = vma->vm_end - PAGE_SIZE; 150 if (pgtable_supports_soft_dirty()) 151 vma_flags_set(&flags, VMA_SOFTDIRTY_BIT); 152 vma->flags = flags; 153 vma->vm_page_prot = vma_get_page_prot(vma); 154 155 err = insert_vm_struct(mm, vma); 156 if (err) 157 goto err; 158 159 mm->stack_vm = mm->total_vm = 1; 160 mmap_write_unlock(mm); 161 *vmap = vma; 162 *top_mem_p = vma->vm_end - sizeof(void *); 163 return 0; 164 165 err: 166 ksm_exit(mm); 167 err_ksm: 168 mmap_write_unlock(mm); 169 err_free: 170 *vmap = NULL; 171 vm_area_free(vma); 172 return err; 173 } 174