1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) 4 */ 5 6 #include <linux/mm.h> 7 #include <linux/module.h> 8 #include <linux/sched/signal.h> 9 10 #include <asm/tlbflush.h> 11 #include <asm/mmu_context.h> 12 #include <as-layout.h> 13 #include <mem_user.h> 14 #include <os.h> 15 #include <skas.h> 16 #include <kern_util.h> 17 18 struct vm_ops { 19 struct mm_id *mm_idp; 20 21 int (*mmap)(struct mm_id *mm_idp, 22 unsigned long virt, unsigned long len, int prot, 23 int phys_fd, unsigned long long offset); 24 int (*unmap)(struct mm_id *mm_idp, 25 unsigned long virt, unsigned long len); 26 int (*mprotect)(struct mm_id *mm_idp, 27 unsigned long virt, unsigned long len, 28 unsigned int prot); 29 }; 30 31 static int kern_map(struct mm_id *mm_idp, 32 unsigned long virt, unsigned long len, int prot, 33 int phys_fd, unsigned long long offset) 34 { 35 /* TODO: Why is executable needed to be always set in the kernel? */ 36 return os_map_memory((void *)virt, phys_fd, offset, len, 37 prot & UM_PROT_READ, prot & UM_PROT_WRITE, 38 1); 39 } 40 41 static int kern_unmap(struct mm_id *mm_idp, 42 unsigned long virt, unsigned long len) 43 { 44 return os_unmap_memory((void *)virt, len); 45 } 46 47 static int kern_mprotect(struct mm_id *mm_idp, 48 unsigned long virt, unsigned long len, 49 unsigned int prot) 50 { 51 return os_protect_memory((void *)virt, len, 52 prot & UM_PROT_READ, prot & UM_PROT_WRITE, 53 1); 54 } 55 56 void report_enomem(void) 57 { 58 printk(KERN_ERR "UML ran out of memory on the host side! " 59 "This can happen due to a memory limitation or " 60 "vm.max_map_count has been reached.\n"); 61 } 62 63 static inline int update_pte_range(pmd_t *pmd, unsigned long addr, 64 unsigned long end, 65 struct vm_ops *ops) 66 { 67 pte_t *pte; 68 int r, w, x, prot, ret = 0; 69 70 pte = pte_offset_kernel(pmd, addr); 71 do { 72 r = pte_read(*pte); 73 w = pte_write(*pte); 74 x = pte_exec(*pte); 75 if (!pte_young(*pte)) { 76 r = 0; 77 w = 0; 78 } else if (!pte_dirty(*pte)) 79 w = 0; 80 81 prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) | 82 (x ? UM_PROT_EXEC : 0)); 83 if (pte_newpage(*pte)) { 84 if (pte_present(*pte)) { 85 if (pte_newpage(*pte)) { 86 __u64 offset; 87 unsigned long phys = 88 pte_val(*pte) & PAGE_MASK; 89 int fd = phys_mapping(phys, &offset); 90 91 ret = ops->mmap(ops->mm_idp, addr, 92 PAGE_SIZE, prot, fd, 93 offset); 94 } 95 } else 96 ret = ops->unmap(ops->mm_idp, addr, PAGE_SIZE); 97 } else if (pte_newprot(*pte)) 98 ret = ops->mprotect(ops->mm_idp, addr, PAGE_SIZE, prot); 99 *pte = pte_mkuptodate(*pte); 100 } while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret)); 101 return ret; 102 } 103 104 static inline int update_pmd_range(pud_t *pud, unsigned long addr, 105 unsigned long end, 106 struct vm_ops *ops) 107 { 108 pmd_t *pmd; 109 unsigned long next; 110 int ret = 0; 111 112 pmd = pmd_offset(pud, addr); 113 do { 114 next = pmd_addr_end(addr, end); 115 if (!pmd_present(*pmd)) { 116 if (pmd_newpage(*pmd)) { 117 ret = ops->unmap(ops->mm_idp, addr, 118 next - addr); 119 pmd_mkuptodate(*pmd); 120 } 121 } 122 else ret = update_pte_range(pmd, addr, next, ops); 123 } while (pmd++, addr = next, ((addr < end) && !ret)); 124 return ret; 125 } 126 127 static inline int update_pud_range(p4d_t *p4d, unsigned long addr, 128 unsigned long end, 129 struct vm_ops *ops) 130 { 131 pud_t *pud; 132 unsigned long next; 133 int ret = 0; 134 135 pud = pud_offset(p4d, addr); 136 do { 137 next = pud_addr_end(addr, end); 138 if (!pud_present(*pud)) { 139 if (pud_newpage(*pud)) { 140 ret = ops->unmap(ops->mm_idp, addr, 141 next - addr); 142 pud_mkuptodate(*pud); 143 } 144 } 145 else ret = update_pmd_range(pud, addr, next, ops); 146 } while (pud++, addr = next, ((addr < end) && !ret)); 147 return ret; 148 } 149 150 static inline int update_p4d_range(pgd_t *pgd, unsigned long addr, 151 unsigned long end, 152 struct vm_ops *ops) 153 { 154 p4d_t *p4d; 155 unsigned long next; 156 int ret = 0; 157 158 p4d = p4d_offset(pgd, addr); 159 do { 160 next = p4d_addr_end(addr, end); 161 if (!p4d_present(*p4d)) { 162 if (p4d_newpage(*p4d)) { 163 ret = ops->unmap(ops->mm_idp, addr, 164 next - addr); 165 p4d_mkuptodate(*p4d); 166 } 167 } else 168 ret = update_pud_range(p4d, addr, next, ops); 169 } while (p4d++, addr = next, ((addr < end) && !ret)); 170 return ret; 171 } 172 173 int um_tlb_sync(struct mm_struct *mm) 174 { 175 pgd_t *pgd; 176 struct vm_ops ops; 177 unsigned long addr = mm->context.sync_tlb_range_from, next; 178 int ret = 0; 179 180 if (mm->context.sync_tlb_range_to == 0) 181 return 0; 182 183 ops.mm_idp = &mm->context.id; 184 if (mm == &init_mm) { 185 ops.mmap = kern_map; 186 ops.unmap = kern_unmap; 187 ops.mprotect = kern_mprotect; 188 } else { 189 ops.mmap = map; 190 ops.unmap = unmap; 191 ops.mprotect = protect; 192 } 193 194 pgd = pgd_offset(mm, addr); 195 do { 196 next = pgd_addr_end(addr, mm->context.sync_tlb_range_to); 197 if (!pgd_present(*pgd)) { 198 if (pgd_newpage(*pgd)) { 199 ret = ops.unmap(ops.mm_idp, addr, 200 next - addr); 201 pgd_mkuptodate(*pgd); 202 } 203 } else 204 ret = update_p4d_range(pgd, addr, next, &ops); 205 } while (pgd++, addr = next, 206 ((addr < mm->context.sync_tlb_range_to) && !ret)); 207 208 if (ret == -ENOMEM) 209 report_enomem(); 210 211 mm->context.sync_tlb_range_from = 0; 212 mm->context.sync_tlb_range_to = 0; 213 214 return ret; 215 } 216 217 void flush_tlb_all(void) 218 { 219 /* 220 * Don't bother flushing if this address space is about to be 221 * destroyed. 222 */ 223 if (atomic_read(¤t->mm->mm_users) == 0) 224 return; 225 226 flush_tlb_mm(current->mm); 227 } 228 229 void flush_tlb_mm(struct mm_struct *mm) 230 { 231 struct vm_area_struct *vma; 232 VMA_ITERATOR(vmi, mm, 0); 233 234 for_each_vma(vmi, vma) 235 um_tlb_mark_sync(mm, vma->vm_start, vma->vm_end); 236 } 237