1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2018 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include <machine/asmacros.h> 32#include <machine/cputypes.h> 33#include <machine/pmap.h> 34#include <machine/specialreg.h> 35 36#include "assym.inc" 37 38/* 39 * Fast path for copyout code. We switch to user space %cr3 and perform 40 * move operation between user memory and copyout buffer, located in the 41 * trampoline area. We must switch to trampoline stack, because both 42 * user and kernel buffer accesses might cause page fault. 43 * 44 * Page fault handler expects %edx to point to the onfault routine. 45 * Handler switches to idlePTD and calls the routine. 46 * The routine must restore the stack, enable interrupts, and 47 * return to the caller, informing it about failure. 48 */ 49 .text 50 51ENTRY(copyout_fast) 52 pushl %ebp 53 movl %esp, %ebp 54 pushl %esi 55 pushl %edi 56 pushl %ebx 57 58 movl 20(%ebp),%ebx /* KCR3 */ 59 /* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */ 60 movl 16(%ebp),%ecx 61 movl 8(%ebp),%esi 62 movl %esp,%eax 63 movl $copyout_fault,%edx 64 65 cli 66 movl PCPU(COPYOUT_BUF),%edi 67pf_y1: rep; movsb 68 69 movl 16(%ebp),%ecx /* len */ 70 movl PCPU(COPYOUT_BUF),%esi /* kaddr */ 71 movl 12(%ebp),%edi /* uaddr */ 72 movl PCPU(TRAMPSTK),%esp 73 movl PCPU(CURPCB),%edx 74 movl PCB_CR3(%edx),%edx /* UCR3 */ 75 movl %edx,%cr3 76 movl $copyout_fault,%edx 77 /* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */ 78pf_x1: rep; movsb 79 80 movl %ebx,%cr3 81 movl %eax,%esp 82 sti 83 xorl %eax,%eax 84 popl %ebx 85 popl %edi 86 popl %esi 87 leave 88 ret 89END(copyout_fast) 90 91ENTRY(copyin_fast) 92 pushl %ebp 93 movl %esp, %ebp 94 pushl %esi 95 pushl %edi 96 pushl %ebx 97 98 movl 20(%ebp),%ebx /* KCR3 */ 99 movl PCPU(CURPCB),%eax 100 movl PCB_CR3(%eax),%edx /* UCR3 */ 101 movl 16(%ebp),%ecx /* len */ 102 movl 8(%ebp),%esi /* udaddr */ 103 movl %esp,%eax 104 105 cli 106 movl PCPU(COPYOUT_BUF),%edi /* kaddr */ 107 movl PCPU(TRAMPSTK),%esp 108 movl %edx,%cr3 109 movl $copyout_fault,%edx 110 /* bcopy(%esi = udaddr, %edi = PCPU(copyout_buf), %ecx = len) */ 111pf_x2: rep; movsb 112 113 movl %ebx,%cr3 114 movl %eax,%esp 115 116 /* bcopy(%esi = PCPU(copyout_buf), %edi = kaddr, %ecx = len) */ 117 movl 16(%ebp),%ecx 118 movl 12(%ebp),%edi 119 movl PCPU(COPYOUT_BUF),%esi 120pf_y2: rep; movsb 121 122 sti 123 xorl %eax,%eax 124 popl %ebx 125 popl %edi 126 popl %esi 127 leave 128 ret 129END(copyin_fast) 130 131 ALIGN_TEXT 132copyout_fault: 133 movl %eax,%esp 134 sti 135 movl $EFAULT,%eax 136 popl %ebx 137 popl %edi 138 popl %esi 139 leave 140 ret 141 142ENTRY(fueword_fast) 143 pushl %ebp 144 movl %esp,%ebp 145 pushl %ebx 146 pushl %esi 147 pushl %edi 148 movl 8(%ebp),%ecx /* from */ 149 movl PCPU(CURPCB),%eax 150 movl PCB_CR3(%eax),%eax 151 movl $fusufault,%edx 152 movl 16(%ebp),%ebx 153 movl %esp,%esi 154 cli 155 movl PCPU(TRAMPSTK),%esp 156 movl %eax,%cr3 157pf_x3: movl (%ecx),%eax 158 movl %ebx,%cr3 159 movl %esi,%esp 160 sti 161 movl 12(%ebp),%edx 162 movl %eax,(%edx) 163 xorl %eax,%eax 164 popl %edi 165 popl %esi 166 popl %ebx 167 leave 168 ret 169END(fueword_fast) 170 171ENTRY(fuword16_fast) 172 pushl %ebp 173 movl %esp,%ebp 174 pushl %ebx 175 pushl %esi 176 pushl %edi 177 movl 8(%ebp),%ecx /* from */ 178 movl PCPU(CURPCB),%eax 179 movl PCB_CR3(%eax),%eax 180 movl $fusufault,%edx 181 movl 12(%ebp),%ebx 182 movl %esp,%esi 183 cli 184 movl PCPU(TRAMPSTK),%esp 185 movl %eax,%cr3 186pf_x4: movzwl (%ecx),%eax 187 movl %ebx,%cr3 188 movl %esi,%esp 189 sti 190 popl %edi 191 popl %esi 192 popl %ebx 193 leave 194 ret 195END(fuword16_fast) 196 197ENTRY(fubyte_fast) 198 pushl %ebp 199 movl %esp,%ebp 200 pushl %ebx 201 pushl %esi 202 pushl %edi 203 movl 8(%ebp),%ecx /* from */ 204 movl PCPU(CURPCB),%eax 205 movl PCB_CR3(%eax),%eax 206 movl $fusufault,%edx 207 movl 12(%ebp),%ebx 208 movl %esp,%esi 209 cli 210 movl PCPU(TRAMPSTK),%esp 211 movl %eax,%cr3 212pf_x5: movzbl (%ecx),%eax 213 movl %ebx,%cr3 214 movl %esi,%esp 215 sti 216 popl %edi 217 popl %esi 218 popl %ebx 219 leave 220 ret 221END(fubyte_fast) 222 223 ALIGN_TEXT 224fusufault: 225 movl %esi,%esp 226 sti 227 xorl %eax,%eax 228 decl %eax 229 popl %edi 230 popl %esi 231 popl %ebx 232 leave 233 ret 234 235ENTRY(suword_fast) 236 pushl %ebp 237 movl %esp,%ebp 238 pushl %ebx 239 pushl %esi 240 pushl %edi 241 movl PCPU(CURPCB),%eax 242 movl PCB_CR3(%eax),%eax 243 movl $fusufault,%edx 244 movl 8(%ebp),%ecx /* to */ 245 movl 12(%ebp),%edi /* val */ 246 movl 16(%ebp),%ebx 247 movl %esp,%esi 248 cli 249 movl PCPU(TRAMPSTK),%esp 250 movl %eax,%cr3 251pf_x6: movl %edi,(%ecx) 252 movl %ebx,%cr3 253 movl %esi,%esp 254 sti 255 xorl %eax,%eax 256 popl %edi 257 popl %esi 258 popl %ebx 259 leave 260 ret 261END(suword_fast) 262 263ENTRY(suword16_fast) 264 pushl %ebp 265 movl %esp,%ebp 266 pushl %ebx 267 pushl %esi 268 pushl %edi 269 movl PCPU(CURPCB),%eax 270 movl PCB_CR3(%eax),%eax 271 movl $fusufault,%edx 272 movl 8(%ebp),%ecx /* to */ 273 movl 12(%ebp),%edi /* val */ 274 movl 16(%ebp),%ebx 275 movl %esp,%esi 276 cli 277 movl PCPU(TRAMPSTK),%esp 278 movl %eax,%cr3 279pf_x7: movw %di,(%ecx) 280 movl %ebx,%cr3 281 movl %esi,%esp 282 sti 283 xorl %eax,%eax 284 popl %edi 285 popl %esi 286 popl %ebx 287 leave 288 ret 289END(suword16_fast) 290 291ENTRY(subyte_fast) 292 pushl %ebp 293 movl %esp,%ebp 294 pushl %ebx 295 pushl %esi 296 pushl %edi 297 movl PCPU(CURPCB),%eax 298 movl PCB_CR3(%eax),%eax 299 movl $fusufault,%edx 300 movl 8(%ebp),%ecx /* to */ 301 movl 12(%ebp),%edi /* val */ 302 movl 16(%ebp),%ebx 303 movl %esp,%esi 304 cli 305 movl PCPU(TRAMPSTK),%esp 306 movl %eax,%cr3 307 movl %edi,%eax 308pf_x8: movb %al,(%ecx) 309 movl %ebx,%cr3 310 movl %esi,%esp 311 sti 312 xorl %eax,%eax 313 popl %edi 314 popl %esi 315 popl %ebx 316 leave 317 ret 318END(subyte_fast) 319