1ccd9b49fSElliott Mitchell/*- 2*4d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 3ccd9b49fSElliott Mitchell * 4ccd9b49fSElliott Mitchell * Copyright (c) 2007 Konstantin Belousov 5ccd9b49fSElliott Mitchell * All rights reserved. 6ccd9b49fSElliott Mitchell * 7ccd9b49fSElliott Mitchell * Redistribution and use in source and binary forms, with or without 8ccd9b49fSElliott Mitchell * modification, are permitted provided that the following conditions 9ccd9b49fSElliott Mitchell * are met: 10ccd9b49fSElliott Mitchell * 1. Redistributions of source code must retain the above copyright 11ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer. 12ccd9b49fSElliott Mitchell * 2. Redistributions in binary form must reproduce the above copyright 13ccd9b49fSElliott Mitchell * notice, this list of conditions and the following disclaimer in the 14ccd9b49fSElliott Mitchell * documentation and/or other materials provided with the distribution. 15ccd9b49fSElliott Mitchell * 16ccd9b49fSElliott Mitchell * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17ccd9b49fSElliott Mitchell * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18ccd9b49fSElliott Mitchell * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19ccd9b49fSElliott Mitchell * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20ccd9b49fSElliott Mitchell * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21ccd9b49fSElliott Mitchell * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22ccd9b49fSElliott Mitchell * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23ccd9b49fSElliott Mitchell * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24ccd9b49fSElliott Mitchell * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25ccd9b49fSElliott Mitchell * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26ccd9b49fSElliott Mitchell * SUCH DAMAGE. 27ccd9b49fSElliott Mitchell */ 28ccd9b49fSElliott Mitchell 29ccd9b49fSElliott Mitchell#include "linux_assym.h" /* system definitions */ 30ccd9b49fSElliott Mitchell#include <machine/asmacros.h> /* miscellaneous asm macros */ 31ccd9b49fSElliott Mitchell#include <machine/specialreg.h> 32ccd9b49fSElliott Mitchell 33ccd9b49fSElliott Mitchell#include "assym.inc" 34ccd9b49fSElliott Mitchell 35ccd9b49fSElliott Mitchellfutex_fault: 36ccd9b49fSElliott Mitchell testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 37ccd9b49fSElliott Mitchell je 1f 38ccd9b49fSElliott Mitchell clac 39ccd9b49fSElliott Mitchell1: movq $0,PCB_ONFAULT(%r8) 40ccd9b49fSElliott Mitchell movl $EFAULT,%eax 41ccd9b49fSElliott Mitchell ret 42ccd9b49fSElliott Mitchell 43ccd9b49fSElliott MitchellENTRY(futex_xchgl_nosmap) 44ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 45ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 46ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 47ccd9b49fSElliott Mitchell cmpq %rax,%rsi 48ccd9b49fSElliott Mitchell ja futex_fault 49ccd9b49fSElliott Mitchell xchgl %edi,(%rsi) 50ccd9b49fSElliott Mitchell movl %edi,(%rdx) 51ccd9b49fSElliott Mitchell xorl %eax,%eax 52ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 53ccd9b49fSElliott Mitchell ret 54ccd9b49fSElliott MitchellEND(futex_xchgl_nosmap) 55ccd9b49fSElliott Mitchell 56ccd9b49fSElliott MitchellENTRY(futex_xchgl_smap) 57ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 58ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 59ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 60ccd9b49fSElliott Mitchell cmpq %rax,%rsi 61ccd9b49fSElliott Mitchell ja futex_fault 62ccd9b49fSElliott Mitchell stac 63ccd9b49fSElliott Mitchell xchgl %edi,(%rsi) 64ccd9b49fSElliott Mitchell clac 65ccd9b49fSElliott Mitchell movl %edi,(%rdx) 66ccd9b49fSElliott Mitchell xorl %eax,%eax 67ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 68ccd9b49fSElliott Mitchell ret 69ccd9b49fSElliott MitchellEND(futex_xchgl_smap) 70ccd9b49fSElliott Mitchell 71ccd9b49fSElliott MitchellENTRY(futex_addl_nosmap) 72ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 73ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 74ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 75ccd9b49fSElliott Mitchell cmpq %rax,%rsi 76ccd9b49fSElliott Mitchell ja futex_fault 77ccd9b49fSElliott Mitchell#ifdef SMP 78ccd9b49fSElliott Mitchell lock 79ccd9b49fSElliott Mitchell#endif 80ccd9b49fSElliott Mitchell xaddl %edi,(%rsi) 81ccd9b49fSElliott Mitchell movl %edi,(%rdx) 82ccd9b49fSElliott Mitchell xorl %eax,%eax 83ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 84ccd9b49fSElliott Mitchell ret 85ccd9b49fSElliott MitchellEND(futex_addl_nosmap) 86ccd9b49fSElliott Mitchell 87ccd9b49fSElliott MitchellENTRY(futex_addl_smap) 88ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 89ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 90ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 91ccd9b49fSElliott Mitchell cmpq %rax,%rsi 92ccd9b49fSElliott Mitchell ja futex_fault 93ccd9b49fSElliott Mitchell stac 94ccd9b49fSElliott Mitchell#ifdef SMP 95ccd9b49fSElliott Mitchell lock 96ccd9b49fSElliott Mitchell#endif 97ccd9b49fSElliott Mitchell xaddl %edi,(%rsi) 98ccd9b49fSElliott Mitchell clac 99ccd9b49fSElliott Mitchell movl %edi,(%rdx) 100ccd9b49fSElliott Mitchell xorl %eax,%eax 101ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 102ccd9b49fSElliott Mitchell ret 103ccd9b49fSElliott MitchellEND(futex_addl_smap) 104ccd9b49fSElliott Mitchell 105ccd9b49fSElliott MitchellENTRY(futex_orl_nosmap) 106ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 107ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 108ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 109ccd9b49fSElliott Mitchell cmpq %rax,%rsi 110ccd9b49fSElliott Mitchell ja futex_fault 111ccd9b49fSElliott Mitchell movl (%rsi),%eax 112ccd9b49fSElliott Mitchell1: movl %eax,%ecx 113ccd9b49fSElliott Mitchell orl %edi,%ecx 114ccd9b49fSElliott Mitchell#ifdef SMP 115ccd9b49fSElliott Mitchell lock 116ccd9b49fSElliott Mitchell#endif 117ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 118ccd9b49fSElliott Mitchell jnz 1b 119ccd9b49fSElliott Mitchell movl %eax,(%rdx) 120ccd9b49fSElliott Mitchell xorl %eax,%eax 121ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 122ccd9b49fSElliott Mitchell ret 123ccd9b49fSElliott MitchellEND(futex_orl_nosmap) 124ccd9b49fSElliott Mitchell 125ccd9b49fSElliott MitchellENTRY(futex_orl_smap) 126ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 127ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 128ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 129ccd9b49fSElliott Mitchell cmpq %rax,%rsi 130ccd9b49fSElliott Mitchell ja futex_fault 131ccd9b49fSElliott Mitchell stac 132ccd9b49fSElliott Mitchell movl (%rsi),%eax 133ccd9b49fSElliott Mitchell1: movl %eax,%ecx 134ccd9b49fSElliott Mitchell orl %edi,%ecx 135ccd9b49fSElliott Mitchell#ifdef SMP 136ccd9b49fSElliott Mitchell lock 137ccd9b49fSElliott Mitchell#endif 138ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 139ccd9b49fSElliott Mitchell jnz 1b 140ccd9b49fSElliott Mitchell clac 141ccd9b49fSElliott Mitchell movl %eax,(%rdx) 142ccd9b49fSElliott Mitchell xorl %eax,%eax 143ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 144ccd9b49fSElliott Mitchell ret 145ccd9b49fSElliott MitchellEND(futex_orl_smap) 146ccd9b49fSElliott Mitchell 147ccd9b49fSElliott MitchellENTRY(futex_andl_nosmap) 148ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 149ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 150ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 151ccd9b49fSElliott Mitchell cmpq %rax,%rsi 152ccd9b49fSElliott Mitchell ja futex_fault 153ccd9b49fSElliott Mitchell movl (%rsi),%eax 154ccd9b49fSElliott Mitchell1: movl %eax,%ecx 155ccd9b49fSElliott Mitchell andl %edi,%ecx 156ccd9b49fSElliott Mitchell#ifdef SMP 157ccd9b49fSElliott Mitchell lock 158ccd9b49fSElliott Mitchell#endif 159ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 160ccd9b49fSElliott Mitchell jnz 1b 161ccd9b49fSElliott Mitchell movl %eax,(%rdx) 162ccd9b49fSElliott Mitchell xorl %eax,%eax 163ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 164ccd9b49fSElliott Mitchell ret 165ccd9b49fSElliott MitchellEND(futex_andl_nosmap) 166ccd9b49fSElliott Mitchell 167ccd9b49fSElliott MitchellENTRY(futex_andl_smap) 168ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 169ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 170ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 171ccd9b49fSElliott Mitchell cmpq %rax,%rsi 172ccd9b49fSElliott Mitchell ja futex_fault 173ccd9b49fSElliott Mitchell stac 174ccd9b49fSElliott Mitchell movl (%rsi),%eax 175ccd9b49fSElliott Mitchell1: movl %eax,%ecx 176ccd9b49fSElliott Mitchell andl %edi,%ecx 177ccd9b49fSElliott Mitchell#ifdef SMP 178ccd9b49fSElliott Mitchell lock 179ccd9b49fSElliott Mitchell#endif 180ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 181ccd9b49fSElliott Mitchell jnz 1b 182ccd9b49fSElliott Mitchell clac 183ccd9b49fSElliott Mitchell movl %eax,(%rdx) 184ccd9b49fSElliott Mitchell xorl %eax,%eax 185ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 186ccd9b49fSElliott Mitchell ret 187ccd9b49fSElliott MitchellEND(futex_andl_smap) 188ccd9b49fSElliott Mitchell 189ccd9b49fSElliott MitchellENTRY(futex_xorl_nosmap) 190ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 191ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 192ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 193ccd9b49fSElliott Mitchell cmpq %rax,%rsi 194ccd9b49fSElliott Mitchell ja futex_fault 195ccd9b49fSElliott Mitchell movl (%rsi),%eax 196ccd9b49fSElliott Mitchell1: movl %eax,%ecx 197ccd9b49fSElliott Mitchell xorl %edi,%ecx 198ccd9b49fSElliott Mitchell#ifdef SMP 199ccd9b49fSElliott Mitchell lock 200ccd9b49fSElliott Mitchell#endif 201ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 202ccd9b49fSElliott Mitchell jnz 1b 203ccd9b49fSElliott Mitchell movl %eax,(%rdx) 204ccd9b49fSElliott Mitchell xorl %eax,%eax 205ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 206ccd9b49fSElliott Mitchell ret 207ccd9b49fSElliott MitchellEND(futex_xorl_nosmap) 208ccd9b49fSElliott Mitchell 209ccd9b49fSElliott MitchellENTRY(futex_xorl_smap) 210ccd9b49fSElliott Mitchell movq PCPU(CURPCB),%r8 211ccd9b49fSElliott Mitchell movq $futex_fault,PCB_ONFAULT(%r8) 212ccd9b49fSElliott Mitchell movq $VM_MAXUSER_ADDRESS-4,%rax 213ccd9b49fSElliott Mitchell cmpq %rax,%rsi 214ccd9b49fSElliott Mitchell ja futex_fault 215ccd9b49fSElliott Mitchell stac 216ccd9b49fSElliott Mitchell movl (%rsi),%eax 217ccd9b49fSElliott Mitchell1: movl %eax,%ecx 218ccd9b49fSElliott Mitchell xorl %edi,%ecx 219ccd9b49fSElliott Mitchell#ifdef SMP 220ccd9b49fSElliott Mitchell lock 221ccd9b49fSElliott Mitchell#endif 222ccd9b49fSElliott Mitchell cmpxchgl %ecx,(%rsi) 223ccd9b49fSElliott Mitchell jnz 1b 224ccd9b49fSElliott Mitchell clac 225ccd9b49fSElliott Mitchell movl %eax,(%rdx) 226ccd9b49fSElliott Mitchell xorl %eax,%eax 227ccd9b49fSElliott Mitchell movq %rax,PCB_ONFAULT(%r8) 228ccd9b49fSElliott Mitchell ret 229ccd9b49fSElliott MitchellEND(futex_xorl_smap) 230