1*ccd9b49fSElliott Mitchell/*- 2*ccd9b49fSElliott Mitchell * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3*ccd9b49fSElliott Mitchell * 4*ccd9b49fSElliott Mitchell * Copyright (c) 2007 Konstantin Belousov 5*ccd9b49fSElliott Mitchell * All rights reserved. 6*ccd9b49fSElliott Mitchell * 7*ccd9b49fSElliott Mitchell * Redistribution and use in source and binary forms, with or without 8*ccd9b49fSElliott Mitchell * modification, are permitted provided that the following conditions 9*ccd9b49fSElliott Mitchell * are met: 10*ccd9b49fSElliott Mitchell * 1. Redistributions of source code must retain the above copyright 11*ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer. 12*ccd9b49fSElliott Mitchell * 2. Redistributions in binary form must reproduce the above copyright 13*ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer in the 14*ccd9b49fSElliott Mitchell * documentation and/or other materials provided with the distribution. 15*ccd9b49fSElliott Mitchell * 16*ccd9b49fSElliott Mitchell * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17*ccd9b49fSElliott Mitchell * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18*ccd9b49fSElliott Mitchell * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19*ccd9b49fSElliott Mitchell * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20*ccd9b49fSElliott Mitchell * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21*ccd9b49fSElliott Mitchell * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22*ccd9b49fSElliott Mitchell * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23*ccd9b49fSElliott Mitchell * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24*ccd9b49fSElliott Mitchell * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25*ccd9b49fSElliott Mitchell * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26*ccd9b49fSElliott Mitchell * SUCH DAMAGE. 27*ccd9b49fSElliott Mitchell * 28*ccd9b49fSElliott Mitchell * $FreeBSD$ 29*ccd9b49fSElliott Mitchell */ 30*ccd9b49fSElliott Mitchell 31*ccd9b49fSElliott Mitchell#include "linux32_assym.h" /* system definitions */ 32*ccd9b49fSElliott Mitchell#include <machine/asmacros.h> /* miscellaneous asm macros */ 33*ccd9b49fSElliott Mitchell#include <machine/specialreg.h> 34*ccd9b49fSElliott Mitchell 35*ccd9b49fSElliott Mitchell#include "assym.inc" 36*ccd9b49fSElliott Mitchell 37*ccd9b49fSElliott Mitchellfutex_fault: 38*ccd9b49fSElliott Mitchell testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 39*ccd9b49fSElliott Mitchell je 1f 40*ccd9b49fSElliott Mitchell clac 41*ccd9b49fSElliott Mitchell1: movq $0,PCB_ONFAULT(%r8) 42*ccd9b49fSElliott Mitchell movl $EFAULT,%eax 43*ccd9b49fSElliott Mitchell ret 44*ccd9b49fSElliott Mitchell 45*ccd9b49fSElliott MitchellENTRY(futex_xchgl_nosmap) 46*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 47*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 48*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 49*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 50*ccd9b49fSElliott Mitchell ja futex_fault 51*ccd9b49fSElliott Mitchell xchgl %edi,(%rsi) 52*ccd9b49fSElliott Mitchell movl %edi,(%rdx) 53*ccd9b49fSElliott Mitchell xorl %eax,%eax 54*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 55*ccd9b49fSElliott Mitchell ret 56*ccd9b49fSElliott MitchellEND(futex_xchgl_nosmap) 57*ccd9b49fSElliott Mitchell 58*ccd9b49fSElliott MitchellENTRY(futex_xchgl_smap) 59*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 60*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 61*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 62*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 63*ccd9b49fSElliott Mitchell ja futex_fault 64*ccd9b49fSElliott Mitchell stac 65*ccd9b49fSElliott Mitchell xchgl %edi,(%rsi) 66*ccd9b49fSElliott Mitchell clac 67*ccd9b49fSElliott Mitchell movl %edi,(%rdx) 68*ccd9b49fSElliott Mitchell xorl %eax,%eax 69*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 70*ccd9b49fSElliott Mitchell ret 71*ccd9b49fSElliott MitchellEND(futex_xchgl_smap) 72*ccd9b49fSElliott Mitchell 73*ccd9b49fSElliott MitchellENTRY(futex_addl_nosmap) 74*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 75*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 76*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 77*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 78*ccd9b49fSElliott Mitchell ja futex_fault 79*ccd9b49fSElliott Mitchell#ifdef SMP 80*ccd9b49fSElliott Mitchell lock 81*ccd9b49fSElliott Mitchell#endif 82*ccd9b49fSElliott Mitchell xaddl %edi,(%rsi) 83*ccd9b49fSElliott Mitchell movl %edi,(%rdx) 84*ccd9b49fSElliott Mitchell xorl %eax,%eax 85*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 86*ccd9b49fSElliott Mitchell ret 87*ccd9b49fSElliott MitchellEND(futex_addl_nosmap) 88*ccd9b49fSElliott Mitchell 89*ccd9b49fSElliott MitchellENTRY(futex_addl_smap) 90*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 91*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 92*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 93*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 94*ccd9b49fSElliott Mitchell ja futex_fault 95*ccd9b49fSElliott Mitchell stac 96*ccd9b49fSElliott Mitchell#ifdef SMP 97*ccd9b49fSElliott Mitchell lock 98*ccd9b49fSElliott Mitchell#endif 99*ccd9b49fSElliott Mitchell xaddl %edi,(%rsi) 100*ccd9b49fSElliott Mitchell clac 101*ccd9b49fSElliott Mitchell movl %edi,(%rdx) 102*ccd9b49fSElliott Mitchell xorl %eax,%eax 103*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 104*ccd9b49fSElliott Mitchell ret 105*ccd9b49fSElliott MitchellEND(futex_addl_smap) 106*ccd9b49fSElliott Mitchell 107*ccd9b49fSElliott MitchellENTRY(futex_orl_nosmap) 108*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 109*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 110*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 111*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 112*ccd9b49fSElliott Mitchell ja futex_fault 113*ccd9b49fSElliott Mitchell movl (%rsi),%eax 114*ccd9b49fSElliott Mitchell1: movl %eax,%ecx 115*ccd9b49fSElliott Mitchell orl %edi,%ecx 116*ccd9b49fSElliott Mitchell#ifdef SMP 117*ccd9b49fSElliott Mitchell lock 118*ccd9b49fSElliott Mitchell#endif 119*ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 120*ccd9b49fSElliott Mitchell jnz 1b 121*ccd9b49fSElliott Mitchell movl %eax,(%rdx) 122*ccd9b49fSElliott Mitchell xorl %eax,%eax 123*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 124*ccd9b49fSElliott Mitchell ret 125*ccd9b49fSElliott MitchellEND(futex_orl_nosmap) 126*ccd9b49fSElliott Mitchell 127*ccd9b49fSElliott MitchellENTRY(futex_orl_smap) 128*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 129*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 130*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 131*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 132*ccd9b49fSElliott Mitchell ja futex_fault 133*ccd9b49fSElliott Mitchell stac 134*ccd9b49fSElliott Mitchell movl (%rsi),%eax 135*ccd9b49fSElliott Mitchell1: movl %eax,%ecx 136*ccd9b49fSElliott Mitchell orl %edi,%ecx 137*ccd9b49fSElliott Mitchell#ifdef SMP 138*ccd9b49fSElliott Mitchell lock 139*ccd9b49fSElliott Mitchell#endif 140*ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 141*ccd9b49fSElliott Mitchell jnz 1b 142*ccd9b49fSElliott Mitchell clac 143*ccd9b49fSElliott Mitchell movl %eax,(%rdx) 144*ccd9b49fSElliott Mitchell xorl %eax,%eax 145*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 146*ccd9b49fSElliott Mitchell ret 147*ccd9b49fSElliott MitchellEND(futex_orl_smap) 148*ccd9b49fSElliott Mitchell 149*ccd9b49fSElliott MitchellENTRY(futex_andl_nosmap) 150*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 151*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 152*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 153*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 154*ccd9b49fSElliott Mitchell ja futex_fault 155*ccd9b49fSElliott Mitchell movl (%rsi),%eax 156*ccd9b49fSElliott Mitchell1: movl %eax,%ecx 157*ccd9b49fSElliott Mitchell andl %edi,%ecx 158*ccd9b49fSElliott Mitchell#ifdef SMP 159*ccd9b49fSElliott Mitchell lock 160*ccd9b49fSElliott Mitchell#endif 161*ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 162*ccd9b49fSElliott Mitchell jnz 1b 163*ccd9b49fSElliott Mitchell movl %eax,(%rdx) 164*ccd9b49fSElliott Mitchell xorl %eax,%eax 165*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 166*ccd9b49fSElliott Mitchell ret 167*ccd9b49fSElliott MitchellEND(futex_andl_nosmap) 168*ccd9b49fSElliott Mitchell 169*ccd9b49fSElliott MitchellENTRY(futex_andl_smap) 170*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 171*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 172*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 173*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 174*ccd9b49fSElliott Mitchell ja futex_fault 175*ccd9b49fSElliott Mitchell stac 176*ccd9b49fSElliott Mitchell movl (%rsi),%eax 177*ccd9b49fSElliott Mitchell1: movl %eax,%ecx 178*ccd9b49fSElliott Mitchell andl %edi,%ecx 179*ccd9b49fSElliott Mitchell#ifdef SMP 180*ccd9b49fSElliott Mitchell lock 181*ccd9b49fSElliott Mitchell#endif 182*ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 183*ccd9b49fSElliott Mitchell jnz 1b 184*ccd9b49fSElliott Mitchell clac 185*ccd9b49fSElliott Mitchell movl %eax,(%rdx) 186*ccd9b49fSElliott Mitchell xorl %eax,%eax 187*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 188*ccd9b49fSElliott Mitchell ret 189*ccd9b49fSElliott MitchellEND(futex_andl_smap) 190*ccd9b49fSElliott Mitchell 191*ccd9b49fSElliott MitchellENTRY(futex_xorl_nosmap) 192*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 193*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 194*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 195*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 196*ccd9b49fSElliott Mitchell ja futex_fault 197*ccd9b49fSElliott Mitchell movl (%rsi),%eax 198*ccd9b49fSElliott Mitchell1: movl %eax,%ecx 199*ccd9b49fSElliott Mitchell xorl %edi,%ecx 200*ccd9b49fSElliott Mitchell#ifdef SMP 201*ccd9b49fSElliott Mitchell lock 202*ccd9b49fSElliott Mitchell#endif 203*ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 204*ccd9b49fSElliott Mitchell jnz 1b 205*ccd9b49fSElliott Mitchell movl %eax,(%rdx) 206*ccd9b49fSElliott Mitchell xorl %eax,%eax 207*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 208*ccd9b49fSElliott Mitchell ret 209*ccd9b49fSElliott MitchellEND(futex_xorl_nosmap) 210*ccd9b49fSElliott Mitchell 211*ccd9b49fSElliott MitchellENTRY(futex_xorl_smap) 212*ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 213*ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 214*ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 215*ccd9b49fSElliott Mitchell cmpq %rax,%rsi 216*ccd9b49fSElliott Mitchell ja futex_fault 217*ccd9b49fSElliott Mitchell stac 218*ccd9b49fSElliott Mitchell movl (%rsi),%eax 219*ccd9b49fSElliott Mitchell1: movl %eax,%ecx 220*ccd9b49fSElliott Mitchell xorl %edi,%ecx 221*ccd9b49fSElliott Mitchell#ifdef SMP 222*ccd9b49fSElliott Mitchell lock 223*ccd9b49fSElliott Mitchell#endif 224*ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 225*ccd9b49fSElliott Mitchell jnz 1b 226*ccd9b49fSElliott Mitchell clac 227*ccd9b49fSElliott Mitchell movl %eax,(%rdx) 228*ccd9b49fSElliott Mitchell xorl %eax,%eax 229*ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 230*ccd9b49fSElliott Mitchell ret 231*ccd9b49fSElliott MitchellEND(futex_xorl_smap) 232