1ccd9b49fSElliott Mitchell/*- 2*4d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 3ccd9b49fSElliott Mitchell * 4ccd9b49fSElliott Mitchell * Copyright (c) 2007 Konstantin Belousov 5ccd9b49fSElliott Mitchell * All rights reserved. 6ccd9b49fSElliott Mitchell * 7ccd9b49fSElliott Mitchell * Redistribution and use in source and binary forms, with or without 8ccd9b49fSElliott Mitchell * modification, are permitted provided that the following conditions 9ccd9b49fSElliott Mitchell * are met: 10ccd9b49fSElliott Mitchell * 1. Redistributions of source code must retain the above copyright 11ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer. 12ccd9b49fSElliott Mitchell * 2. Redistributions in binary form must reproduce the above copyright 13ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer in the 14ccd9b49fSElliott Mitchell * documentation and/or other materials provided with the distribution. 15ccd9b49fSElliott Mitchell * 16ccd9b49fSElliott Mitchell * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17ccd9b49fSElliott Mitchell * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18ccd9b49fSElliott Mitchell * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19ccd9b49fSElliott Mitchell * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20ccd9b49fSElliott Mitchell * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21ccd9b49fSElliott Mitchell * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22ccd9b49fSElliott Mitchell * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23ccd9b49fSElliott Mitchell * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24ccd9b49fSElliott Mitchell * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25ccd9b49fSElliott Mitchell * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26ccd9b49fSElliott Mitchell * SUCH DAMAGE. 27ccd9b49fSElliott Mitchell * 28ccd9b49fSElliott Mitchell * $FreeBSD$ 29ccd9b49fSElliott Mitchell */ 30ccd9b49fSElliott Mitchell 31ccd9b49fSElliott Mitchell#include "linux32_assym.h" /* system definitions */ 32ccd9b49fSElliott Mitchell#include <machine/asmacros.h> /* miscellaneous asm macros */ 33ccd9b49fSElliott Mitchell#include <machine/specialreg.h> 34ccd9b49fSElliott Mitchell 35ccd9b49fSElliott Mitchell#include "assym.inc" 36ccd9b49fSElliott Mitchell 37ccd9b49fSElliott Mitchellfutex_fault: 38ccd9b49fSElliott Mitchell testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 39ccd9b49fSElliott Mitchell je 1f 40ccd9b49fSElliott Mitchell clac 41ccd9b49fSElliott Mitchell1: movq $0,PCB_ONFAULT(%r8) 42ccd9b49fSElliott Mitchell movl $EFAULT,%eax 43ccd9b49fSElliott Mitchell ret 44ccd9b49fSElliott Mitchell 45ccd9b49fSElliott MitchellENTRY(futex_xchgl_nosmap) 46ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 47ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 48ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 49ccd9b49fSElliott Mitchell cmpq %rax,%rsi 50ccd9b49fSElliott Mitchell ja futex_fault 51ccd9b49fSElliott Mitchell xchgl %edi,(%rsi) 52ccd9b49fSElliott Mitchell movl %edi,(%rdx) 53ccd9b49fSElliott Mitchell xorl %eax,%eax 54ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 55ccd9b49fSElliott Mitchell ret 56ccd9b49fSElliott MitchellEND(futex_xchgl_nosmap) 57ccd9b49fSElliott Mitchell 58ccd9b49fSElliott MitchellENTRY(futex_xchgl_smap) 59ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 60ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 61ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 62ccd9b49fSElliott Mitchell cmpq %rax,%rsi 63ccd9b49fSElliott Mitchell ja futex_fault 64ccd9b49fSElliott Mitchell stac 65ccd9b49fSElliott Mitchell xchgl %edi,(%rsi) 66ccd9b49fSElliott Mitchell clac 67ccd9b49fSElliott Mitchell movl %edi,(%rdx) 68ccd9b49fSElliott Mitchell xorl %eax,%eax 69ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 70ccd9b49fSElliott Mitchell ret 71ccd9b49fSElliott MitchellEND(futex_xchgl_smap) 72ccd9b49fSElliott Mitchell 73ccd9b49fSElliott MitchellENTRY(futex_addl_nosmap) 74ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 75ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 76ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 77ccd9b49fSElliott Mitchell cmpq %rax,%rsi 78ccd9b49fSElliott Mitchell ja futex_fault 79ccd9b49fSElliott Mitchell#ifdef SMP 80ccd9b49fSElliott Mitchell lock 81ccd9b49fSElliott Mitchell#endif 82ccd9b49fSElliott Mitchell xaddl %edi,(%rsi) 83ccd9b49fSElliott Mitchell movl %edi,(%rdx) 84ccd9b49fSElliott Mitchell xorl %eax,%eax 85ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 86ccd9b49fSElliott Mitchell ret 87ccd9b49fSElliott MitchellEND(futex_addl_nosmap) 88ccd9b49fSElliott Mitchell 89ccd9b49fSElliott MitchellENTRY(futex_addl_smap) 90ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 91ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 92ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 93ccd9b49fSElliott Mitchell cmpq %rax,%rsi 94ccd9b49fSElliott Mitchell ja futex_fault 95ccd9b49fSElliott Mitchell stac 96ccd9b49fSElliott Mitchell#ifdef SMP 97ccd9b49fSElliott Mitchell lock 98ccd9b49fSElliott Mitchell#endif 99ccd9b49fSElliott Mitchell xaddl %edi,(%rsi) 100ccd9b49fSElliott Mitchell clac 101ccd9b49fSElliott Mitchell movl %edi,(%rdx) 102ccd9b49fSElliott Mitchell xorl %eax,%eax 103ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 104ccd9b49fSElliott Mitchell ret 105ccd9b49fSElliott MitchellEND(futex_addl_smap) 106ccd9b49fSElliott Mitchell 107ccd9b49fSElliott MitchellENTRY(futex_orl_nosmap) 108ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 109ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 110ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 111ccd9b49fSElliott Mitchell cmpq %rax,%rsi 112ccd9b49fSElliott Mitchell ja futex_fault 113ccd9b49fSElliott Mitchell movl (%rsi),%eax 114ccd9b49fSElliott Mitchell1: movl %eax,%ecx 115ccd9b49fSElliott Mitchell orl %edi,%ecx 116ccd9b49fSElliott Mitchell#ifdef SMP 117ccd9b49fSElliott Mitchell lock 118ccd9b49fSElliott Mitchell#endif 119ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 120ccd9b49fSElliott Mitchell jnz 1b 121ccd9b49fSElliott Mitchell movl %eax,(%rdx) 122ccd9b49fSElliott Mitchell xorl %eax,%eax 123ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 124ccd9b49fSElliott Mitchell ret 125ccd9b49fSElliott MitchellEND(futex_orl_nosmap) 126ccd9b49fSElliott Mitchell 127ccd9b49fSElliott MitchellENTRY(futex_orl_smap) 128ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 129ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 130ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 131ccd9b49fSElliott Mitchell cmpq %rax,%rsi 132ccd9b49fSElliott Mitchell ja futex_fault 133ccd9b49fSElliott Mitchell stac 134ccd9b49fSElliott Mitchell movl (%rsi),%eax 135ccd9b49fSElliott Mitchell1: movl %eax,%ecx 136ccd9b49fSElliott Mitchell orl %edi,%ecx 137ccd9b49fSElliott Mitchell#ifdef SMP 138ccd9b49fSElliott Mitchell lock 139ccd9b49fSElliott Mitchell#endif 140ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 141ccd9b49fSElliott Mitchell jnz 1b 142ccd9b49fSElliott Mitchell clac 143ccd9b49fSElliott Mitchell movl %eax,(%rdx) 144ccd9b49fSElliott Mitchell xorl %eax,%eax 145ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 146ccd9b49fSElliott Mitchell ret 147ccd9b49fSElliott MitchellEND(futex_orl_smap) 148ccd9b49fSElliott Mitchell 149ccd9b49fSElliott MitchellENTRY(futex_andl_nosmap) 150ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 151ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 152ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 153ccd9b49fSElliott Mitchell cmpq %rax,%rsi 154ccd9b49fSElliott Mitchell ja futex_fault 155ccd9b49fSElliott Mitchell movl (%rsi),%eax 156ccd9b49fSElliott Mitchell1: movl %eax,%ecx 157ccd9b49fSElliott Mitchell andl %edi,%ecx 158ccd9b49fSElliott Mitchell#ifdef SMP 159ccd9b49fSElliott Mitchell lock 160ccd9b49fSElliott Mitchell#endif 161ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 162ccd9b49fSElliott Mitchell jnz 1b 163ccd9b49fSElliott Mitchell movl %eax,(%rdx) 164ccd9b49fSElliott Mitchell xorl %eax,%eax 165ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 166ccd9b49fSElliott Mitchell ret 167ccd9b49fSElliott MitchellEND(futex_andl_nosmap) 168ccd9b49fSElliott Mitchell 169ccd9b49fSElliott MitchellENTRY(futex_andl_smap) 170ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 171ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 172ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 173ccd9b49fSElliott Mitchell cmpq %rax,%rsi 174ccd9b49fSElliott Mitchell ja futex_fault 175ccd9b49fSElliott Mitchell stac 176ccd9b49fSElliott Mitchell movl (%rsi),%eax 177ccd9b49fSElliott Mitchell1: movl %eax,%ecx 178ccd9b49fSElliott Mitchell andl %edi,%ecx 179ccd9b49fSElliott Mitchell#ifdef SMP 180ccd9b49fSElliott Mitchell lock 181ccd9b49fSElliott Mitchell#endif 182ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 183ccd9b49fSElliott Mitchell jnz 1b 184ccd9b49fSElliott Mitchell clac 185ccd9b49fSElliott Mitchell movl %eax,(%rdx) 186ccd9b49fSElliott Mitchell xorl %eax,%eax 187ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 188ccd9b49fSElliott Mitchell ret 189ccd9b49fSElliott MitchellEND(futex_andl_smap) 190ccd9b49fSElliott Mitchell 191ccd9b49fSElliott MitchellENTRY(futex_xorl_nosmap) 192ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 193ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 194ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 195ccd9b49fSElliott Mitchell cmpq %rax,%rsi 196ccd9b49fSElliott Mitchell ja futex_fault 197ccd9b49fSElliott Mitchell movl (%rsi),%eax 198ccd9b49fSElliott Mitchell1: movl %eax,%ecx 199ccd9b49fSElliott Mitchell xorl %edi,%ecx 200ccd9b49fSElliott Mitchell#ifdef SMP 201ccd9b49fSElliott Mitchell lock 202ccd9b49fSElliott Mitchell#endif 203ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 204ccd9b49fSElliott Mitchell jnz 1b 205ccd9b49fSElliott Mitchell movl %eax,(%rdx) 206ccd9b49fSElliott Mitchell xorl %eax,%eax 207ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 208ccd9b49fSElliott Mitchell ret 209ccd9b49fSElliott MitchellEND(futex_xorl_nosmap) 210ccd9b49fSElliott Mitchell 211ccd9b49fSElliott MitchellENTRY(futex_xorl_smap) 212ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 213ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 214ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 215ccd9b49fSElliott Mitchell cmpq %rax,%rsi 216ccd9b49fSElliott Mitchell ja futex_fault 217ccd9b49fSElliott Mitchell stac 218ccd9b49fSElliott Mitchell movl (%rsi),%eax 219ccd9b49fSElliott Mitchell1: movl %eax,%ecx 220ccd9b49fSElliott Mitchell xorl %edi,%ecx 221ccd9b49fSElliott Mitchell#ifdef SMP 222ccd9b49fSElliott Mitchell lock 223ccd9b49fSElliott Mitchell#endif 224ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 225ccd9b49fSElliott Mitchell jnz 1b 226ccd9b49fSElliott Mitchell clac 227ccd9b49fSElliott Mitchell movl %eax,(%rdx) 228ccd9b49fSElliott Mitchell xorl %eax,%eax 229ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 230ccd9b49fSElliott Mitchell ret 231ccd9b49fSElliott MitchellEND(futex_xorl_smap) 232