1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2007 Konstantin Belousov 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31#include "linux_assym.h" /* system definitions */ 32#include <machine/asmacros.h> /* miscellaneous asm macros */ 33#include <machine/specialreg.h> 34 35#include "assym.inc" 36 37futex_fault: 38 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 39 je 1f 40 clac 411: movq $0,PCB_ONFAULT(%r8) 42 movl $EFAULT,%eax 43 ret 44 45ENTRY(futex_xchgl_nosmap) 46 movq PCPU(CURPCB),%r8 47 movq $futex_fault,PCB_ONFAULT(%r8) 48 movq $VM_MAXUSER_ADDRESS-4,%rax 49 cmpq %rax,%rsi 50 ja futex_fault 51 xchgl %edi,(%rsi) 52 movl %edi,(%rdx) 53 xorl %eax,%eax 54 movq %rax,PCB_ONFAULT(%r8) 55 ret 56END(futex_xchgl_nosmap) 57 58ENTRY(futex_xchgl_smap) 59 movq PCPU(CURPCB),%r8 60 movq $futex_fault,PCB_ONFAULT(%r8) 61 movq $VM_MAXUSER_ADDRESS-4,%rax 62 cmpq %rax,%rsi 63 ja futex_fault 64 stac 65 xchgl %edi,(%rsi) 66 clac 67 movl %edi,(%rdx) 68 xorl %eax,%eax 69 movq %rax,PCB_ONFAULT(%r8) 70 ret 71END(futex_xchgl_smap) 72 73ENTRY(futex_addl_nosmap) 74 movq PCPU(CURPCB),%r8 75 movq $futex_fault,PCB_ONFAULT(%r8) 76 movq $VM_MAXUSER_ADDRESS-4,%rax 77 cmpq %rax,%rsi 78 ja futex_fault 79#ifdef SMP 80 lock 81#endif 82 xaddl %edi,(%rsi) 83 movl %edi,(%rdx) 84 xorl %eax,%eax 85 movq %rax,PCB_ONFAULT(%r8) 86 ret 87END(futex_addl_nosmap) 88 89ENTRY(futex_addl_smap) 90 movq PCPU(CURPCB),%r8 91 movq $futex_fault,PCB_ONFAULT(%r8) 92 movq $VM_MAXUSER_ADDRESS-4,%rax 93 cmpq %rax,%rsi 94 ja futex_fault 95 stac 96#ifdef SMP 97 lock 98#endif 99 xaddl %edi,(%rsi) 100 clac 101 movl %edi,(%rdx) 102 xorl %eax,%eax 103 movq %rax,PCB_ONFAULT(%r8) 104 ret 105END(futex_addl_smap) 106 107ENTRY(futex_orl_nosmap) 108 movq PCPU(CURPCB),%r8 109 movq $futex_fault,PCB_ONFAULT(%r8) 110 movq $VM_MAXUSER_ADDRESS-4,%rax 111 cmpq %rax,%rsi 112 ja futex_fault 113 movl (%rsi),%eax 1141: movl %eax,%ecx 115 orl %edi,%ecx 116#ifdef SMP 117 lock 118#endif 119 cmpxchgl %ecx,(%rsi) 120 jnz 1b 121 movl %eax,(%rdx) 122 xorl %eax,%eax 123 movq %rax,PCB_ONFAULT(%r8) 124 ret 125END(futex_orl_nosmap) 126 127ENTRY(futex_orl_smap) 128 movq PCPU(CURPCB),%r8 129 movq $futex_fault,PCB_ONFAULT(%r8) 130 movq $VM_MAXUSER_ADDRESS-4,%rax 131 cmpq %rax,%rsi 132 ja futex_fault 133 stac 134 movl (%rsi),%eax 1351: movl %eax,%ecx 136 orl %edi,%ecx 137#ifdef SMP 138 lock 139#endif 140 cmpxchgl %ecx,(%rsi) 141 jnz 1b 142 clac 143 movl %eax,(%rdx) 144 xorl %eax,%eax 145 movq %rax,PCB_ONFAULT(%r8) 146 ret 147END(futex_orl_smap) 148 149ENTRY(futex_andl_nosmap) 150 movq PCPU(CURPCB),%r8 151 movq $futex_fault,PCB_ONFAULT(%r8) 152 movq $VM_MAXUSER_ADDRESS-4,%rax 153 cmpq %rax,%rsi 154 ja futex_fault 155 movl (%rsi),%eax 1561: movl %eax,%ecx 157 andl %edi,%ecx 158#ifdef SMP 159 lock 160#endif 161 cmpxchgl %ecx,(%rsi) 162 jnz 1b 163 movl %eax,(%rdx) 164 xorl %eax,%eax 165 movq %rax,PCB_ONFAULT(%r8) 166 ret 167END(futex_andl_nosmap) 168 169ENTRY(futex_andl_smap) 170 movq PCPU(CURPCB),%r8 171 movq $futex_fault,PCB_ONFAULT(%r8) 172 movq $VM_MAXUSER_ADDRESS-4,%rax 173 cmpq %rax,%rsi 174 ja futex_fault 175 stac 176 movl (%rsi),%eax 1771: movl %eax,%ecx 178 andl %edi,%ecx 179#ifdef SMP 180 lock 181#endif 182 cmpxchgl %ecx,(%rsi) 183 jnz 1b 184 clac 185 movl %eax,(%rdx) 186 xorl %eax,%eax 187 movq %rax,PCB_ONFAULT(%r8) 188 ret 189END(futex_andl_smap) 190 191ENTRY(futex_xorl_nosmap) 192 movq PCPU(CURPCB),%r8 193 movq $futex_fault,PCB_ONFAULT(%r8) 194 movq $VM_MAXUSER_ADDRESS-4,%rax 195 cmpq %rax,%rsi 196 ja futex_fault 197 movl (%rsi),%eax 1981: movl %eax,%ecx 199 xorl %edi,%ecx 200#ifdef SMP 201 lock 202#endif 203 cmpxchgl %ecx,(%rsi) 204 jnz 1b 205 movl %eax,(%rdx) 206 xorl %eax,%eax 207 movq %rax,PCB_ONFAULT(%r8) 208 ret 209END(futex_xorl_nosmap) 210 211ENTRY(futex_xorl_smap) 212 movq PCPU(CURPCB),%r8 213 movq $futex_fault,PCB_ONFAULT(%r8) 214 movq $VM_MAXUSER_ADDRESS-4,%rax 215 cmpq %rax,%rsi 216 ja futex_fault 217 stac 218 movl (%rsi),%eax 2191: movl %eax,%ecx 220 xorl %edi,%ecx 221#ifdef SMP 222 lock 223#endif 224 cmpxchgl %ecx,(%rsi) 225 jnz 1b 226 clac 227 movl %eax,(%rdx) 228 xorl %eax,%eax 229 movq %rax,PCB_ONFAULT(%r8) 230 ret 231END(futex_xorl_smap) 232