1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2007 Konstantin Belousov 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include "linux_assym.h" /* system definitions */ 30#include <machine/asmacros.h> /* miscellaneous asm macros */ 31#include <machine/specialreg.h> 32 33#include "assym.inc" 34 35futex_fault: 36 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 37 je 1f 38 clac 391: movq $0,PCB_ONFAULT(%r8) 40 movl $EFAULT,%eax 41 ret 42 43ENTRY(futex_xchgl_nosmap) 44 movq PCPU(CURPCB),%r8 45 movq $futex_fault,PCB_ONFAULT(%r8) 46 movq $VM_MAXUSER_ADDRESS-4,%rax 47 cmpq %rax,%rsi 48 ja futex_fault 49 xchgl %edi,(%rsi) 50 movl %edi,(%rdx) 51 xorl %eax,%eax 52 movq %rax,PCB_ONFAULT(%r8) 53 ret 54END(futex_xchgl_nosmap) 55 56ENTRY(futex_xchgl_smap) 57 movq PCPU(CURPCB),%r8 58 movq $futex_fault,PCB_ONFAULT(%r8) 59 movq $VM_MAXUSER_ADDRESS-4,%rax 60 cmpq %rax,%rsi 61 ja futex_fault 62 stac 63 xchgl %edi,(%rsi) 64 clac 65 movl %edi,(%rdx) 66 xorl %eax,%eax 67 movq %rax,PCB_ONFAULT(%r8) 68 ret 69END(futex_xchgl_smap) 70 71ENTRY(futex_addl_nosmap) 72 movq PCPU(CURPCB),%r8 73 movq $futex_fault,PCB_ONFAULT(%r8) 74 movq $VM_MAXUSER_ADDRESS-4,%rax 75 cmpq %rax,%rsi 76 ja futex_fault 77#ifdef SMP 78 lock 79#endif 80 xaddl %edi,(%rsi) 81 movl %edi,(%rdx) 82 xorl %eax,%eax 83 movq %rax,PCB_ONFAULT(%r8) 84 ret 85END(futex_addl_nosmap) 86 87ENTRY(futex_addl_smap) 88 movq PCPU(CURPCB),%r8 89 movq $futex_fault,PCB_ONFAULT(%r8) 90 movq $VM_MAXUSER_ADDRESS-4,%rax 91 cmpq %rax,%rsi 92 ja futex_fault 93 stac 94#ifdef SMP 95 lock 96#endif 97 xaddl %edi,(%rsi) 98 clac 99 movl %edi,(%rdx) 100 xorl %eax,%eax 101 movq %rax,PCB_ONFAULT(%r8) 102 ret 103END(futex_addl_smap) 104 105ENTRY(futex_orl_nosmap) 106 movq PCPU(CURPCB),%r8 107 movq $futex_fault,PCB_ONFAULT(%r8) 108 movq $VM_MAXUSER_ADDRESS-4,%rax 109 cmpq %rax,%rsi 110 ja futex_fault 111 movl (%rsi),%eax 1121: movl %eax,%ecx 113 orl %edi,%ecx 114#ifdef SMP 115 lock 116#endif 117 cmpxchgl %ecx,(%rsi) 118 jnz 1b 119 movl %eax,(%rdx) 120 xorl %eax,%eax 121 movq %rax,PCB_ONFAULT(%r8) 122 ret 123END(futex_orl_nosmap) 124 125ENTRY(futex_orl_smap) 126 movq PCPU(CURPCB),%r8 127 movq $futex_fault,PCB_ONFAULT(%r8) 128 movq $VM_MAXUSER_ADDRESS-4,%rax 129 cmpq %rax,%rsi 130 ja futex_fault 131 stac 132 movl (%rsi),%eax 1331: movl %eax,%ecx 134 orl %edi,%ecx 135#ifdef SMP 136 lock 137#endif 138 cmpxchgl %ecx,(%rsi) 139 jnz 1b 140 clac 141 movl %eax,(%rdx) 142 xorl %eax,%eax 143 movq %rax,PCB_ONFAULT(%r8) 144 ret 145END(futex_orl_smap) 146 147ENTRY(futex_andl_nosmap) 148 movq PCPU(CURPCB),%r8 149 movq $futex_fault,PCB_ONFAULT(%r8) 150 movq $VM_MAXUSER_ADDRESS-4,%rax 151 cmpq %rax,%rsi 152 ja futex_fault 153 movl (%rsi),%eax 1541: movl %eax,%ecx 155 andl %edi,%ecx 156#ifdef SMP 157 lock 158#endif 159 cmpxchgl %ecx,(%rsi) 160 jnz 1b 161 movl %eax,(%rdx) 162 xorl %eax,%eax 163 movq %rax,PCB_ONFAULT(%r8) 164 ret 165END(futex_andl_nosmap) 166 167ENTRY(futex_andl_smap) 168 movq PCPU(CURPCB),%r8 169 movq $futex_fault,PCB_ONFAULT(%r8) 170 movq $VM_MAXUSER_ADDRESS-4,%rax 171 cmpq %rax,%rsi 172 ja futex_fault 173 stac 174 movl (%rsi),%eax 1751: movl %eax,%ecx 176 andl %edi,%ecx 177#ifdef SMP 178 lock 179#endif 180 cmpxchgl %ecx,(%rsi) 181 jnz 1b 182 clac 183 movl %eax,(%rdx) 184 xorl %eax,%eax 185 movq %rax,PCB_ONFAULT(%r8) 186 ret 187END(futex_andl_smap) 188 189ENTRY(futex_xorl_nosmap) 190 movq PCPU(CURPCB),%r8 191 movq $futex_fault,PCB_ONFAULT(%r8) 192 movq $VM_MAXUSER_ADDRESS-4,%rax 193 cmpq %rax,%rsi 194 ja futex_fault 195 movl (%rsi),%eax 1961: movl %eax,%ecx 197 xorl %edi,%ecx 198#ifdef SMP 199 lock 200#endif 201 cmpxchgl %ecx,(%rsi) 202 jnz 1b 203 movl %eax,(%rdx) 204 xorl %eax,%eax 205 movq %rax,PCB_ONFAULT(%r8) 206 ret 207END(futex_xorl_nosmap) 208 209ENTRY(futex_xorl_smap) 210 movq PCPU(CURPCB),%r8 211 movq $futex_fault,PCB_ONFAULT(%r8) 212 movq $VM_MAXUSER_ADDRESS-4,%rax 213 cmpq %rax,%rsi 214 ja futex_fault 215 stac 216 movl (%rsi),%eax 2171: movl %eax,%ecx 218 xorl %edi,%ecx 219#ifdef SMP 220 lock 221#endif 222 cmpxchgl %ecx,(%rsi) 223 jnz 1b 224 clac 225 movl %eax,(%rdx) 226 xorl %eax,%eax 227 movq %rax,PCB_ONFAULT(%r8) 228 ret 229END(futex_xorl_smap) 230