1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * $FreeBSD$ 34 */ 35 36 /* 37 * Functions to provide access to special i386 instructions. 38 */ 39 40 #ifndef _MACHINE_CPUFUNC_H_ 41 #define _MACHINE_CPUFUNC_H_ 42 43 #define readb(va) (*(volatile u_int8_t *) (va)) 44 #define readw(va) (*(volatile u_int16_t *) (va)) 45 #define readl(va) (*(volatile u_int32_t *) (va)) 46 47 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 48 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 49 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 50 51 #ifdef __GNUC__ 52 53 #ifdef SMP 54 #include <machine/lock.h> /* XXX */ 55 #endif 56 57 #ifdef SWTCH_OPTIM_STATS 58 extern int tlb_flush_count; /* XXX */ 59 #endif 60 61 static __inline void 62 breakpoint(void) 63 { 64 __asm __volatile("int $3"); 65 } 66 67 static __inline u_int 68 bsfl(u_int mask) 69 { 70 u_int result; 71 72 __asm __volatile("bsfl %0,%0" : "=r" (result) : "0" (mask)); 73 return (result); 74 } 75 76 static __inline u_int 77 bsrl(u_int mask) 78 { 79 u_int result; 80 81 __asm __volatile("bsrl %0,%0" : "=r" (result) : "0" (mask)); 82 return (result); 83 } 84 85 static __inline void 86 disable_intr(void) 87 { 88 __asm __volatile("cli" : : : "memory"); 89 #ifdef SMP 90 MPINTR_LOCK(); 91 #endif 92 } 93 94 static __inline void 95 enable_intr(void) 96 { 97 #ifdef SMP 98 MPINTR_UNLOCK(); 99 #endif 100 __asm __volatile("sti"); 101 } 102 103 #define HAVE_INLINE_FFS 104 105 static __inline int 106 ffs(int mask) 107 { 108 /* 109 * Note that gcc-2's builtin ffs would be used if we didn't declare 110 * this inline or turn off the builtin. The builtin is faster but 111 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 112 * versions. 113 */ 114 return (mask == 0 ? mask : bsfl((u_int)mask) + 1); 115 } 116 117 #define HAVE_INLINE_FLS 118 119 static __inline int 120 fls(int mask) 121 { 122 return (mask == 0 ? mask : bsrl((u_int)mask) + 1); 123 } 124 125 #if __GNUC__ < 2 126 127 #define inb(port) inbv(port) 128 #define outb(port, data) outbv(port, data) 129 130 #else /* __GNUC >= 2 */ 131 132 /* 133 * The following complications are to get around gcc not having a 134 * constraint letter for the range 0..255. We still put "d" in the 135 * constraint because "i" isn't a valid constraint when the port 136 * isn't constant. This only matters for -O0 because otherwise 137 * the non-working version gets optimized away. 138 * 139 * Use an expression-statement instead of a conditional expression 140 * because gcc-2.6.0 would promote the operands of the conditional 141 * and produce poor code for "if ((inb(var) & const1) == const2)". 142 * 143 * The unnecessary test `(port) < 0x10000' is to generate a warning if 144 * the `port' has type u_short or smaller. Such types are pessimal. 145 * This actually only works for signed types. The range check is 146 * careful to avoid generating warnings. 147 */ 148 #define inb(port) __extension__ ({ \ 149 u_char _data; \ 150 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 151 && (port) < 0x10000) \ 152 _data = inbc(port); \ 153 else \ 154 _data = inbv(port); \ 155 _data; }) 156 157 #define outb(port, data) ( \ 158 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 159 && (port) < 0x10000 \ 160 ? outbc(port, data) : outbv(port, data)) 161 162 static __inline u_char 163 inbc(u_int port) 164 { 165 u_char data; 166 167 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 168 return (data); 169 } 170 171 static __inline void 172 outbc(u_int port, u_char data) 173 { 174 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 175 } 176 177 #endif /* __GNUC <= 2 */ 178 179 static __inline u_char 180 inbv(u_int port) 181 { 182 u_char data; 183 /* 184 * We use %%dx and not %1 here because i/o is done at %dx and not at 185 * %edx, while gcc generates inferior code (movw instead of movl) 186 * if we tell it to load (u_short) port. 187 */ 188 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 189 return (data); 190 } 191 192 static __inline u_int 193 inl(u_int port) 194 { 195 u_int data; 196 197 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 198 return (data); 199 } 200 201 static __inline void 202 insb(u_int port, void *addr, size_t cnt) 203 { 204 __asm __volatile("cld; rep; insb" 205 : "=D" (addr), "=c" (cnt) 206 : "0" (addr), "1" (cnt), "d" (port) 207 : "memory"); 208 } 209 210 static __inline void 211 insw(u_int port, void *addr, size_t cnt) 212 { 213 __asm __volatile("cld; rep; insw" 214 : "=D" (addr), "=c" (cnt) 215 : "0" (addr), "1" (cnt), "d" (port) 216 : "memory"); 217 } 218 219 static __inline void 220 insl(u_int port, void *addr, size_t cnt) 221 { 222 __asm __volatile("cld; rep; insl" 223 : "=D" (addr), "=c" (cnt) 224 : "0" (addr), "1" (cnt), "d" (port) 225 : "memory"); 226 } 227 228 static __inline void 229 invd(void) 230 { 231 __asm __volatile("invd"); 232 } 233 234 #if defined(SMP) && defined(_KERNEL) 235 236 /* 237 * When using APIC IPI's, invlpg() is not simply the invlpg instruction 238 * (this is a bug) and the inlining cost is prohibitive since the call 239 * executes into the IPI transmission system. 240 */ 241 void invlpg __P((u_int addr)); 242 void invltlb __P((void)); 243 244 static __inline void 245 cpu_invlpg(void *addr) 246 { 247 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 248 } 249 250 static __inline void 251 cpu_invltlb(void) 252 { 253 u_int temp; 254 /* 255 * This should be implemented as load_cr3(rcr3()) when load_cr3() 256 * is inlined. 257 */ 258 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 259 : : "memory"); 260 #if defined(SWTCH_OPTIM_STATS) 261 ++tlb_flush_count; 262 #endif 263 } 264 265 #else /* !(SMP && _KERNEL) */ 266 267 static __inline void 268 invlpg(u_int addr) 269 { 270 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 271 } 272 273 static __inline void 274 invltlb(void) 275 { 276 u_int temp; 277 /* 278 * This should be implemented as load_cr3(rcr3()) when load_cr3() 279 * is inlined. 280 */ 281 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 282 : : "memory"); 283 #ifdef SWTCH_OPTIM_STATS 284 ++tlb_flush_count; 285 #endif 286 } 287 288 #endif /* SMP && _KERNEL */ 289 290 static __inline u_short 291 inw(u_int port) 292 { 293 u_short data; 294 295 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 296 return (data); 297 } 298 299 static __inline u_int 300 loadandclear(volatile u_int *addr) 301 { 302 u_int result; 303 304 __asm __volatile("xorl %0,%0; xchgl %1,%0" 305 : "=&r" (result) : "m" (*addr)); 306 return (result); 307 } 308 309 static __inline void 310 outbv(u_int port, u_char data) 311 { 312 u_char al; 313 /* 314 * Use an unnecessary assignment to help gcc's register allocator. 315 * This make a large difference for gcc-1.40 and a tiny difference 316 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 317 * best results. gcc-2.6.0 can't handle this. 318 */ 319 al = data; 320 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 321 } 322 323 static __inline void 324 outl(u_int port, u_int data) 325 { 326 /* 327 * outl() and outw() aren't used much so we haven't looked at 328 * possible micro-optimizations such as the unnecessary 329 * assignment for them. 330 */ 331 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 332 } 333 334 static __inline void 335 outsb(u_int port, const void *addr, size_t cnt) 336 { 337 __asm __volatile("cld; rep; outsb" 338 : "=S" (addr), "=c" (cnt) 339 : "0" (addr), "1" (cnt), "d" (port)); 340 } 341 342 static __inline void 343 outsw(u_int port, const void *addr, size_t cnt) 344 { 345 __asm __volatile("cld; rep; outsw" 346 : "=S" (addr), "=c" (cnt) 347 : "0" (addr), "1" (cnt), "d" (port)); 348 } 349 350 static __inline void 351 outsl(u_int port, const void *addr, size_t cnt) 352 { 353 __asm __volatile("cld; rep; outsl" 354 : "=S" (addr), "=c" (cnt) 355 : "0" (addr), "1" (cnt), "d" (port)); 356 } 357 358 static __inline void 359 outw(u_int port, u_short data) 360 { 361 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 362 } 363 364 static __inline u_int 365 rcr2(void) 366 { 367 u_int data; 368 369 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 370 return (data); 371 } 372 373 static __inline u_int 374 read_eflags(void) 375 { 376 u_int ef; 377 378 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 379 return (ef); 380 } 381 382 static __inline u_int64_t 383 rdmsr(u_int msr) 384 { 385 u_int64_t rv; 386 387 __asm __volatile(".byte 0x0f, 0x32" : "=A" (rv) : "c" (msr)); 388 return (rv); 389 } 390 391 static __inline u_int64_t 392 rdpmc(u_int pmc) 393 { 394 u_int64_t rv; 395 396 __asm __volatile(".byte 0x0f, 0x33" : "=A" (rv) : "c" (pmc)); 397 return (rv); 398 } 399 400 static __inline u_int64_t 401 rdtsc(void) 402 { 403 u_int64_t rv; 404 405 __asm __volatile(".byte 0x0f, 0x31" : "=A" (rv)); 406 return (rv); 407 } 408 409 static __inline void 410 wbinvd(void) 411 { 412 __asm __volatile("wbinvd"); 413 } 414 415 static __inline void 416 write_eflags(u_int ef) 417 { 418 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 419 } 420 421 static __inline void 422 wrmsr(u_int msr, u_int64_t newval) 423 { 424 __asm __volatile(".byte 0x0f, 0x30" : : "A" (newval), "c" (msr)); 425 } 426 427 static __inline u_int 428 rfs(void) 429 { 430 u_int sel; 431 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 432 return (sel); 433 } 434 435 static __inline u_int 436 rgs(void) 437 { 438 u_int sel; 439 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 440 return (sel); 441 } 442 443 static __inline void 444 load_fs(u_int sel) 445 { 446 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 447 } 448 449 static __inline void 450 load_gs(u_int sel) 451 { 452 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 453 } 454 455 static __inline u_int 456 rdr0(void) 457 { 458 u_int data; 459 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 460 return (data); 461 } 462 463 static __inline u_int 464 rdr1(void) 465 { 466 u_int data; 467 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 468 return (data); 469 } 470 471 static __inline u_int 472 rdr2(void) 473 { 474 u_int data; 475 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 476 return (data); 477 } 478 479 static __inline u_int 480 rdr3(void) 481 { 482 u_int data; 483 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 484 return (data); 485 } 486 487 static __inline u_int 488 rdr6(void) 489 { 490 u_int data; 491 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 492 return (data); 493 } 494 495 static __inline u_int 496 rdr7(void) 497 { 498 u_int data; 499 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 500 return (data); 501 } 502 503 #else /* !__GNUC__ */ 504 505 int breakpoint __P((void)); 506 u_int bsfl __P((u_int mask)); 507 u_int bsrl __P((u_int mask)); 508 void disable_intr __P((void)); 509 void enable_intr __P((void)); 510 u_char inb __P((u_int port)); 511 u_int inl __P((u_int port)); 512 void insb __P((u_int port, void *addr, size_t cnt)); 513 void insl __P((u_int port, void *addr, size_t cnt)); 514 void insw __P((u_int port, void *addr, size_t cnt)); 515 void invd __P((void)); 516 void invlpg __P((u_int addr)); 517 void invltlb __P((void)); 518 u_short inw __P((u_int port)); 519 u_int loadandclear __P((u_int *addr)); 520 void outb __P((u_int port, u_char data)); 521 void outl __P((u_int port, u_int data)); 522 void outsb __P((u_int port, void *addr, size_t cnt)); 523 void outsl __P((u_int port, void *addr, size_t cnt)); 524 void outsw __P((u_int port, void *addr, size_t cnt)); 525 void outw __P((u_int port, u_short data)); 526 u_int rcr2 __P((void)); 527 u_int64_t rdmsr __P((u_int msr)); 528 u_int64_t rdpmc __P((u_int pmc)); 529 u_int64_t rdtsc __P((void)); 530 u_int read_eflags __P((void)); 531 void wbinvd __P((void)); 532 void write_eflags __P((u_int ef)); 533 void wrmsr __P((u_int msr, u_int64_t newval)); 534 u_int rfs __P((void)); 535 u_int rgs __P((void)); 536 void load_fs __P((u_int sel)); 537 void load_gs __P((u_int sel)); 538 539 #endif /* __GNUC__ */ 540 541 void load_cr0 __P((u_int cr0)); 542 void load_cr3 __P((u_int cr3)); 543 void load_cr4 __P((u_int cr4)); 544 void ltr __P((u_short sel)); 545 u_int rcr0 __P((void)); 546 u_int rcr3 __P((void)); 547 u_int rcr4 __P((void)); 548 void load_dr6 __P((u_int dr6)); 549 void reset_dbregs __P((void)); 550 551 #endif /* !_MACHINE_CPUFUNC_H_ */ 552