1ccd9b49fSElliott Mitchell/*- 2*4d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 3ccd9b49fSElliott Mitchell * 4ccd9b49fSElliott Mitchell * Copyright (c) 2018 The FreeBSD Foundation 5ccd9b49fSElliott Mitchell * 6ccd9b49fSElliott Mitchell * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7ccd9b49fSElliott Mitchell * under sponsorship from the FreeBSD Foundation. 8ccd9b49fSElliott Mitchell * 9ccd9b49fSElliott Mitchell * Redistribution and use in source and binary forms, with or without 10ccd9b49fSElliott Mitchell * modification, are permitted provided that the following conditions 11ccd9b49fSElliott Mitchell * are met: 12ccd9b49fSElliott Mitchell * 1. Redistributions of source code must retain the above copyright 13ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer. 14ccd9b49fSElliott Mitchell * 2. Redistributions in binary form must reproduce the above copyright 15ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer in the 16ccd9b49fSElliott Mitchell * documentation and/or other materials provided with the distribution. 17ccd9b49fSElliott Mitchell * 18ccd9b49fSElliott Mitchell * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19ccd9b49fSElliott Mitchell * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20ccd9b49fSElliott Mitchell * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21ccd9b49fSElliott Mitchell * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22ccd9b49fSElliott Mitchell * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23ccd9b49fSElliott Mitchell * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24ccd9b49fSElliott Mitchell * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25ccd9b49fSElliott Mitchell * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26ccd9b49fSElliott Mitchell * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27ccd9b49fSElliott Mitchell * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28ccd9b49fSElliott Mitchell * SUCH DAMAGE. 29ccd9b49fSElliott Mitchell */ 30ccd9b49fSElliott Mitchell 31ccd9b49fSElliott Mitchell#include <machine/asmacros.h> 32ccd9b49fSElliott Mitchell#include <machine/cputypes.h> 33ccd9b49fSElliott Mitchell#include <machine/pmap.h> 34ccd9b49fSElliott Mitchell#include <machine/specialreg.h> 35ccd9b49fSElliott Mitchell 36ccd9b49fSElliott Mitchell#include "assym.inc" 37ccd9b49fSElliott Mitchell 38ccd9b49fSElliott Mitchell/* 39ccd9b49fSElliott Mitchell * Fast path for copyout code. We switch to user space %cr3 and perform 40ccd9b49fSElliott Mitchell * move operation between user memory and copyout buffer, located in the 41ccd9b49fSElliott Mitchell * trampoline area. We must switch to trampoline stack, because both 42ccd9b49fSElliott Mitchell * user and kernel buffer accesses might cause page fault. 43ccd9b49fSElliott Mitchell * 44ccd9b49fSElliott Mitchell * Page fault handler expects %edx to point to the onfault routine. 45ccd9b49fSElliott Mitchell * Handler switches to idlePTD and calls the routine. 46ccd9b49fSElliott Mitchell * The routine must restore the stack, enable interrupts, and 47ccd9b49fSElliott Mitchell * return to the caller, informing it about failure. 48ccd9b49fSElliott Mitchell */ 49ccd9b49fSElliott Mitchell .text 50ccd9b49fSElliott Mitchell 51ccd9b49fSElliott MitchellENTRY(copyout_fast) 52ccd9b49fSElliott Mitchell pushl %ebp 53ccd9b49fSElliott Mitchell movl %esp, %ebp 54ccd9b49fSElliott Mitchell pushl %esi 55ccd9b49fSElliott Mitchell pushl %edi 56ccd9b49fSElliott Mitchell pushl %ebx 57ccd9b49fSElliott Mitchell 58ccd9b49fSElliott Mitchell movl 20(%ebp),%ebx /* KCR3 */ 59ccd9b49fSElliott Mitchell /* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */ 60ccd9b49fSElliott Mitchell movl 16(%ebp),%ecx 61ccd9b49fSElliott Mitchell movl 8(%ebp),%esi 62ccd9b49fSElliott Mitchell movl %esp,%eax 63ccd9b49fSElliott Mitchell movl $copyout_fault,%edx 64ccd9b49fSElliott Mitchell 65ccd9b49fSElliott Mitchell cli 66ccd9b49fSElliott Mitchell movl PCPU(COPYOUT_BUF),%edi 67ccd9b49fSElliott Mitchellpf_y1: rep; movsb 68ccd9b49fSElliott Mitchell 69ccd9b49fSElliott Mitchell movl 16(%ebp),%ecx /* len */ 70ccd9b49fSElliott Mitchell movl PCPU(COPYOUT_BUF),%esi /* kaddr */ 71ccd9b49fSElliott Mitchell movl 12(%ebp),%edi /* uaddr */ 72ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 73ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%edx 74ccd9b49fSElliott Mitchell movl PCB_CR3(%edx),%edx /* UCR3 */ 75ccd9b49fSElliott Mitchell movl %edx,%cr3 76ccd9b49fSElliott Mitchell movl $copyout_fault,%edx 77ccd9b49fSElliott Mitchell /* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */ 78ccd9b49fSElliott Mitchellpf_x1: rep; movsb 79ccd9b49fSElliott Mitchell 80ccd9b49fSElliott Mitchell movl %ebx,%cr3 81ccd9b49fSElliott Mitchell movl %eax,%esp 82ccd9b49fSElliott Mitchell sti 83ccd9b49fSElliott Mitchell xorl %eax,%eax 84ccd9b49fSElliott Mitchell popl %ebx 85ccd9b49fSElliott Mitchell popl %edi 86ccd9b49fSElliott Mitchell popl %esi 87ccd9b49fSElliott Mitchell leave 88ccd9b49fSElliott Mitchell ret 89ccd9b49fSElliott MitchellEND(copyout_fast) 90ccd9b49fSElliott Mitchell 91ccd9b49fSElliott MitchellENTRY(copyin_fast) 92ccd9b49fSElliott Mitchell pushl %ebp 93ccd9b49fSElliott Mitchell movl %esp, %ebp 94ccd9b49fSElliott Mitchell pushl %esi 95ccd9b49fSElliott Mitchell pushl %edi 96ccd9b49fSElliott Mitchell pushl %ebx 97ccd9b49fSElliott Mitchell 98ccd9b49fSElliott Mitchell movl 20(%ebp),%ebx /* KCR3 */ 99ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%eax 100ccd9b49fSElliott Mitchell movl PCB_CR3(%eax),%edx /* UCR3 */ 101ccd9b49fSElliott Mitchell movl 16(%ebp),%ecx /* len */ 102ccd9b49fSElliott Mitchell movl 8(%ebp),%esi /* udaddr */ 103ccd9b49fSElliott Mitchell movl %esp,%eax 104ccd9b49fSElliott Mitchell 105ccd9b49fSElliott Mitchell cli 106ccd9b49fSElliott Mitchell movl PCPU(COPYOUT_BUF),%edi /* kaddr */ 107ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 108ccd9b49fSElliott Mitchell movl %edx,%cr3 109ccd9b49fSElliott Mitchell movl $copyout_fault,%edx 110ccd9b49fSElliott Mitchell /* bcopy(%esi = udaddr, %edi = PCPU(copyout_buf), %ecx = len) */ 111ccd9b49fSElliott Mitchellpf_x2: rep; movsb 112ccd9b49fSElliott Mitchell 113ccd9b49fSElliott Mitchell movl %ebx,%cr3 114ccd9b49fSElliott Mitchell movl %eax,%esp 115ccd9b49fSElliott Mitchell 116ccd9b49fSElliott Mitchell /* bcopy(%esi = PCPU(copyout_buf), %edi = kaddr, %ecx = len) */ 117ccd9b49fSElliott Mitchell movl 16(%ebp),%ecx 118ccd9b49fSElliott Mitchell movl 12(%ebp),%edi 119ccd9b49fSElliott Mitchell movl PCPU(COPYOUT_BUF),%esi 120ccd9b49fSElliott Mitchellpf_y2: rep; movsb 121ccd9b49fSElliott Mitchell 122ccd9b49fSElliott Mitchell sti 123ccd9b49fSElliott Mitchell xorl %eax,%eax 124ccd9b49fSElliott Mitchell popl %ebx 125ccd9b49fSElliott Mitchell popl %edi 126ccd9b49fSElliott Mitchell popl %esi 127ccd9b49fSElliott Mitchell leave 128ccd9b49fSElliott Mitchell ret 129ccd9b49fSElliott MitchellEND(copyin_fast) 130ccd9b49fSElliott Mitchell 131ccd9b49fSElliott Mitchell ALIGN_TEXT 132ccd9b49fSElliott Mitchellcopyout_fault: 133ccd9b49fSElliott Mitchell movl %eax,%esp 134ccd9b49fSElliott Mitchell sti 135ccd9b49fSElliott Mitchell movl $EFAULT,%eax 136ccd9b49fSElliott Mitchell popl %ebx 137ccd9b49fSElliott Mitchell popl %edi 138ccd9b49fSElliott Mitchell popl %esi 139ccd9b49fSElliott Mitchell leave 140ccd9b49fSElliott Mitchell ret 141ccd9b49fSElliott Mitchell 142ccd9b49fSElliott MitchellENTRY(fueword_fast) 143ccd9b49fSElliott Mitchell pushl %ebp 144ccd9b49fSElliott Mitchell movl %esp,%ebp 145ccd9b49fSElliott Mitchell pushl %ebx 146ccd9b49fSElliott Mitchell pushl %esi 147ccd9b49fSElliott Mitchell pushl %edi 148ccd9b49fSElliott Mitchell movl 8(%ebp),%ecx /* from */ 149ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%eax 150ccd9b49fSElliott Mitchell movl PCB_CR3(%eax),%eax 151ccd9b49fSElliott Mitchell movl $fusufault,%edx 152ccd9b49fSElliott Mitchell movl 16(%ebp),%ebx 153ccd9b49fSElliott Mitchell movl %esp,%esi 154ccd9b49fSElliott Mitchell cli 155ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 156ccd9b49fSElliott Mitchell movl %eax,%cr3 157ccd9b49fSElliott Mitchellpf_x3: movl (%ecx),%eax 158ccd9b49fSElliott Mitchell movl %ebx,%cr3 159ccd9b49fSElliott Mitchell movl %esi,%esp 160ccd9b49fSElliott Mitchell sti 161ccd9b49fSElliott Mitchell movl 12(%ebp),%edx 162ccd9b49fSElliott Mitchell movl %eax,(%edx) 163ccd9b49fSElliott Mitchell xorl %eax,%eax 164ccd9b49fSElliott Mitchell popl %edi 165ccd9b49fSElliott Mitchell popl %esi 166ccd9b49fSElliott Mitchell popl %ebx 167ccd9b49fSElliott Mitchell leave 168ccd9b49fSElliott Mitchell ret 169ccd9b49fSElliott MitchellEND(fueword_fast) 170ccd9b49fSElliott Mitchell 171ccd9b49fSElliott MitchellENTRY(fuword16_fast) 172ccd9b49fSElliott Mitchell pushl %ebp 173ccd9b49fSElliott Mitchell movl %esp,%ebp 174ccd9b49fSElliott Mitchell pushl %ebx 175ccd9b49fSElliott Mitchell pushl %esi 176ccd9b49fSElliott Mitchell pushl %edi 177ccd9b49fSElliott Mitchell movl 8(%ebp),%ecx /* from */ 178ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%eax 179ccd9b49fSElliott Mitchell movl PCB_CR3(%eax),%eax 180ccd9b49fSElliott Mitchell movl $fusufault,%edx 181ccd9b49fSElliott Mitchell movl 12(%ebp),%ebx 182ccd9b49fSElliott Mitchell movl %esp,%esi 183ccd9b49fSElliott Mitchell cli 184ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 185ccd9b49fSElliott Mitchell movl %eax,%cr3 186ccd9b49fSElliott Mitchellpf_x4: movzwl (%ecx),%eax 187ccd9b49fSElliott Mitchell movl %ebx,%cr3 188ccd9b49fSElliott Mitchell movl %esi,%esp 189ccd9b49fSElliott Mitchell sti 190ccd9b49fSElliott Mitchell popl %edi 191ccd9b49fSElliott Mitchell popl %esi 192ccd9b49fSElliott Mitchell popl %ebx 193ccd9b49fSElliott Mitchell leave 194ccd9b49fSElliott Mitchell ret 195ccd9b49fSElliott MitchellEND(fuword16_fast) 196ccd9b49fSElliott Mitchell 197ccd9b49fSElliott MitchellENTRY(fubyte_fast) 198ccd9b49fSElliott Mitchell pushl %ebp 199ccd9b49fSElliott Mitchell movl %esp,%ebp 200ccd9b49fSElliott Mitchell pushl %ebx 201ccd9b49fSElliott Mitchell pushl %esi 202ccd9b49fSElliott Mitchell pushl %edi 203ccd9b49fSElliott Mitchell movl 8(%ebp),%ecx /* from */ 204ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%eax 205ccd9b49fSElliott Mitchell movl PCB_CR3(%eax),%eax 206ccd9b49fSElliott Mitchell movl $fusufault,%edx 207ccd9b49fSElliott Mitchell movl 12(%ebp),%ebx 208ccd9b49fSElliott Mitchell movl %esp,%esi 209ccd9b49fSElliott Mitchell cli 210ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 211ccd9b49fSElliott Mitchell movl %eax,%cr3 212ccd9b49fSElliott Mitchellpf_x5: movzbl (%ecx),%eax 213ccd9b49fSElliott Mitchell movl %ebx,%cr3 214ccd9b49fSElliott Mitchell movl %esi,%esp 215ccd9b49fSElliott Mitchell sti 216ccd9b49fSElliott Mitchell popl %edi 217ccd9b49fSElliott Mitchell popl %esi 218ccd9b49fSElliott Mitchell popl %ebx 219ccd9b49fSElliott Mitchell leave 220ccd9b49fSElliott Mitchell ret 221ccd9b49fSElliott MitchellEND(fubyte_fast) 222ccd9b49fSElliott Mitchell 223ccd9b49fSElliott Mitchell ALIGN_TEXT 224ccd9b49fSElliott Mitchellfusufault: 225ccd9b49fSElliott Mitchell movl %esi,%esp 226ccd9b49fSElliott Mitchell sti 227ccd9b49fSElliott Mitchell xorl %eax,%eax 228ccd9b49fSElliott Mitchell decl %eax 229ccd9b49fSElliott Mitchell popl %edi 230ccd9b49fSElliott Mitchell popl %esi 231ccd9b49fSElliott Mitchell popl %ebx 232ccd9b49fSElliott Mitchell leave 233ccd9b49fSElliott Mitchell ret 234ccd9b49fSElliott Mitchell 235ccd9b49fSElliott MitchellENTRY(suword_fast) 236ccd9b49fSElliott Mitchell pushl %ebp 237ccd9b49fSElliott Mitchell movl %esp,%ebp 238ccd9b49fSElliott Mitchell pushl %ebx 239ccd9b49fSElliott Mitchell pushl %esi 240ccd9b49fSElliott Mitchell pushl %edi 241ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%eax 242ccd9b49fSElliott Mitchell movl PCB_CR3(%eax),%eax 243ccd9b49fSElliott Mitchell movl $fusufault,%edx 244ccd9b49fSElliott Mitchell movl 8(%ebp),%ecx /* to */ 245ccd9b49fSElliott Mitchell movl 12(%ebp),%edi /* val */ 246ccd9b49fSElliott Mitchell movl 16(%ebp),%ebx 247ccd9b49fSElliott Mitchell movl %esp,%esi 248ccd9b49fSElliott Mitchell cli 249ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 250ccd9b49fSElliott Mitchell movl %eax,%cr3 251ccd9b49fSElliott Mitchellpf_x6: movl %edi,(%ecx) 252ccd9b49fSElliott Mitchell movl %ebx,%cr3 253ccd9b49fSElliott Mitchell movl %esi,%esp 254ccd9b49fSElliott Mitchell sti 255ccd9b49fSElliott Mitchell xorl %eax,%eax 256ccd9b49fSElliott Mitchell popl %edi 257ccd9b49fSElliott Mitchell popl %esi 258ccd9b49fSElliott Mitchell popl %ebx 259ccd9b49fSElliott Mitchell leave 260ccd9b49fSElliott Mitchell ret 261ccd9b49fSElliott MitchellEND(suword_fast) 262ccd9b49fSElliott Mitchell 263ccd9b49fSElliott MitchellENTRY(suword16_fast) 264ccd9b49fSElliott Mitchell pushl %ebp 265ccd9b49fSElliott Mitchell movl %esp,%ebp 266ccd9b49fSElliott Mitchell pushl %ebx 267ccd9b49fSElliott Mitchell pushl %esi 268ccd9b49fSElliott Mitchell pushl %edi 269ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%eax 270ccd9b49fSElliott Mitchell movl PCB_CR3(%eax),%eax 271ccd9b49fSElliott Mitchell movl $fusufault,%edx 272ccd9b49fSElliott Mitchell movl 8(%ebp),%ecx /* to */ 273ccd9b49fSElliott Mitchell movl 12(%ebp),%edi /* val */ 274ccd9b49fSElliott Mitchell movl 16(%ebp),%ebx 275ccd9b49fSElliott Mitchell movl %esp,%esi 276ccd9b49fSElliott Mitchell cli 277ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 278ccd9b49fSElliott Mitchell movl %eax,%cr3 279ccd9b49fSElliott Mitchellpf_x7: movw %di,(%ecx) 280ccd9b49fSElliott Mitchell movl %ebx,%cr3 281ccd9b49fSElliott Mitchell movl %esi,%esp 282ccd9b49fSElliott Mitchell sti 283ccd9b49fSElliott Mitchell xorl %eax,%eax 284ccd9b49fSElliott Mitchell popl %edi 285ccd9b49fSElliott Mitchell popl %esi 286ccd9b49fSElliott Mitchell popl %ebx 287ccd9b49fSElliott Mitchell leave 288ccd9b49fSElliott Mitchell ret 289ccd9b49fSElliott MitchellEND(suword16_fast) 290ccd9b49fSElliott Mitchell 291ccd9b49fSElliott MitchellENTRY(subyte_fast) 292ccd9b49fSElliott Mitchell pushl %ebp 293ccd9b49fSElliott Mitchell movl %esp,%ebp 294ccd9b49fSElliott Mitchell pushl %ebx 295ccd9b49fSElliott Mitchell pushl %esi 296ccd9b49fSElliott Mitchell pushl %edi 297ccd9b49fSElliott Mitchell movl PCPU(CURPCB),%eax 298ccd9b49fSElliott Mitchell movl PCB_CR3(%eax),%eax 299ccd9b49fSElliott Mitchell movl $fusufault,%edx 300ccd9b49fSElliott Mitchell movl 8(%ebp),%ecx /* to */ 301ccd9b49fSElliott Mitchell movl 12(%ebp),%edi /* val */ 302ccd9b49fSElliott Mitchell movl 16(%ebp),%ebx 303ccd9b49fSElliott Mitchell movl %esp,%esi 304ccd9b49fSElliott Mitchell cli 305ccd9b49fSElliott Mitchell movl PCPU(TRAMPSTK),%esp 306ccd9b49fSElliott Mitchell movl %eax,%cr3 307ccd9b49fSElliott Mitchell movl %edi,%eax 308ccd9b49fSElliott Mitchellpf_x8: movb %al,(%ecx) 309ccd9b49fSElliott Mitchell movl %ebx,%cr3 310ccd9b49fSElliott Mitchell movl %esi,%esp 311ccd9b49fSElliott Mitchell sti 312ccd9b49fSElliott Mitchell xorl %eax,%eax 313ccd9b49fSElliott Mitchell popl %edi 314ccd9b49fSElliott Mitchell popl %esi 315ccd9b49fSElliott Mitchell popl %ebx 316ccd9b49fSElliott Mitchell leave 317ccd9b49fSElliott Mitchell ret 318ccd9b49fSElliott MitchellEND(subyte_fast) 319