1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * $FreeBSD$ 34 */ 35 36 /* 37 * Functions to provide access to special i386 instructions. 38 * This in included in sys/systm.h, and that file should be 39 * used in preference to this. 40 */ 41 42 #ifndef _MACHINE_CPUFUNC_H_ 43 #define _MACHINE_CPUFUNC_H_ 44 45 #include <sys/cdefs.h> 46 #include <machine/psl.h> 47 48 __BEGIN_DECLS 49 #define readb(va) (*(volatile u_int8_t *) (va)) 50 #define readw(va) (*(volatile u_int16_t *) (va)) 51 #define readl(va) (*(volatile u_int32_t *) (va)) 52 53 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 54 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 55 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 56 57 #define CRITICAL_FORK (read_eflags() | PSL_I) 58 59 #ifdef __GNUC__ 60 61 #ifdef SWTCH_OPTIM_STATS 62 extern int tlb_flush_count; /* XXX */ 63 #endif 64 65 static __inline void 66 breakpoint(void) 67 { 68 __asm __volatile("int $3"); 69 } 70 71 static __inline u_int 72 bsfl(u_int mask) 73 { 74 u_int result; 75 76 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 77 return (result); 78 } 79 80 static __inline u_int 81 bsrl(u_int mask) 82 { 83 u_int result; 84 85 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 86 return (result); 87 } 88 89 static __inline void 90 disable_intr(void) 91 { 92 __asm __volatile("cli" : : : "memory"); 93 } 94 95 static __inline void 96 enable_intr(void) 97 { 98 __asm __volatile("sti"); 99 } 100 101 #define HAVE_INLINE_FFS 102 103 static __inline int 104 ffs(int mask) 105 { 106 /* 107 * Note that gcc-2's builtin ffs would be used if we didn't declare 108 * this inline or turn off the builtin. The builtin is faster but 109 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 110 * versions. 111 */ 112 return (mask == 0 ? mask : bsfl((u_int)mask) + 1); 113 } 114 115 #define HAVE_INLINE_FLS 116 117 static __inline int 118 fls(int mask) 119 { 120 return (mask == 0 ? mask : bsrl((u_int)mask) + 1); 121 } 122 123 #if __GNUC__ < 2 124 125 #define inb(port) inbv(port) 126 #define outb(port, data) outbv(port, data) 127 128 #else /* __GNUC >= 2 */ 129 130 /* 131 * The following complications are to get around gcc not having a 132 * constraint letter for the range 0..255. We still put "d" in the 133 * constraint because "i" isn't a valid constraint when the port 134 * isn't constant. This only matters for -O0 because otherwise 135 * the non-working version gets optimized away. 136 * 137 * Use an expression-statement instead of a conditional expression 138 * because gcc-2.6.0 would promote the operands of the conditional 139 * and produce poor code for "if ((inb(var) & const1) == const2)". 140 * 141 * The unnecessary test `(port) < 0x10000' is to generate a warning if 142 * the `port' has type u_short or smaller. Such types are pessimal. 143 * This actually only works for signed types. The range check is 144 * careful to avoid generating warnings. 145 */ 146 #define inb(port) __extension__ ({ \ 147 u_char _data; \ 148 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 149 && (port) < 0x10000) \ 150 _data = inbc(port); \ 151 else \ 152 _data = inbv(port); \ 153 _data; }) 154 155 #define outb(port, data) ( \ 156 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 157 && (port) < 0x10000 \ 158 ? outbc(port, data) : outbv(port, data)) 159 160 static __inline u_char 161 inbc(u_int port) 162 { 163 u_char data; 164 165 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 166 return (data); 167 } 168 169 static __inline void 170 outbc(u_int port, u_char data) 171 { 172 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 173 } 174 175 #endif /* __GNUC <= 2 */ 176 177 static __inline u_char 178 inbv(u_int port) 179 { 180 u_char data; 181 /* 182 * We use %%dx and not %1 here because i/o is done at %dx and not at 183 * %edx, while gcc generates inferior code (movw instead of movl) 184 * if we tell it to load (u_short) port. 185 */ 186 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 187 return (data); 188 } 189 190 static __inline u_int 191 inl(u_int port) 192 { 193 u_int data; 194 195 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 196 return (data); 197 } 198 199 static __inline void 200 insb(u_int port, void *addr, size_t cnt) 201 { 202 __asm __volatile("cld; rep; insb" 203 : "+D" (addr), "+c" (cnt) 204 : "d" (port) 205 : "memory"); 206 } 207 208 static __inline void 209 insw(u_int port, void *addr, size_t cnt) 210 { 211 __asm __volatile("cld; rep; insw" 212 : "+D" (addr), "+c" (cnt) 213 : "d" (port) 214 : "memory"); 215 } 216 217 static __inline void 218 insl(u_int port, void *addr, size_t cnt) 219 { 220 __asm __volatile("cld; rep; insl" 221 : "+D" (addr), "+c" (cnt) 222 : "d" (port) 223 : "memory"); 224 } 225 226 static __inline void 227 invd(void) 228 { 229 __asm __volatile("invd"); 230 } 231 232 #if defined(SMP) && defined(_KERNEL) 233 234 /* 235 * When using APIC IPI's, invlpg() is not simply the invlpg instruction 236 * (this is a bug) and the inlining cost is prohibitive since the call 237 * executes into the IPI transmission system. 238 */ 239 void invlpg(u_int addr); 240 void invltlb(void); 241 242 static __inline void 243 cpu_invlpg(void *addr) 244 { 245 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 246 } 247 248 static __inline void 249 cpu_invltlb(void) 250 { 251 u_int temp; 252 /* 253 * This should be implemented as load_cr3(rcr3()) when load_cr3() 254 * is inlined. 255 */ 256 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 257 : : "memory"); 258 #if defined(SWTCH_OPTIM_STATS) 259 ++tlb_flush_count; 260 #endif 261 } 262 263 #else /* !(SMP && _KERNEL) */ 264 265 static __inline void 266 invlpg(u_int addr) 267 { 268 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 269 } 270 271 static __inline void 272 invltlb(void) 273 { 274 u_int temp; 275 /* 276 * This should be implemented as load_cr3(rcr3()) when load_cr3() 277 * is inlined. 278 */ 279 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 280 : : "memory"); 281 #ifdef SWTCH_OPTIM_STATS 282 ++tlb_flush_count; 283 #endif 284 } 285 286 #endif /* SMP && _KERNEL */ 287 288 static __inline u_short 289 inw(u_int port) 290 { 291 u_short data; 292 293 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 294 return (data); 295 } 296 297 static __inline void 298 outbv(u_int port, u_char data) 299 { 300 u_char al; 301 /* 302 * Use an unnecessary assignment to help gcc's register allocator. 303 * This make a large difference for gcc-1.40 and a tiny difference 304 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 305 * best results. gcc-2.6.0 can't handle this. 306 */ 307 al = data; 308 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 309 } 310 311 static __inline void 312 outl(u_int port, u_int data) 313 { 314 /* 315 * outl() and outw() aren't used much so we haven't looked at 316 * possible micro-optimizations such as the unnecessary 317 * assignment for them. 318 */ 319 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 320 } 321 322 static __inline void 323 outsb(u_int port, const void *addr, size_t cnt) 324 { 325 __asm __volatile("cld; rep; outsb" 326 : "+S" (addr), "+c" (cnt) 327 : "d" (port)); 328 } 329 330 static __inline void 331 outsw(u_int port, const void *addr, size_t cnt) 332 { 333 __asm __volatile("cld; rep; outsw" 334 : "+S" (addr), "+c" (cnt) 335 : "d" (port)); 336 } 337 338 static __inline void 339 outsl(u_int port, const void *addr, size_t cnt) 340 { 341 __asm __volatile("cld; rep; outsl" 342 : "+S" (addr), "+c" (cnt) 343 : "d" (port)); 344 } 345 346 static __inline void 347 outw(u_int port, u_short data) 348 { 349 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 350 } 351 352 static __inline u_int 353 rcr2(void) 354 { 355 u_int data; 356 357 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 358 return (data); 359 } 360 361 static __inline u_int 362 read_eflags(void) 363 { 364 u_int ef; 365 366 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 367 return (ef); 368 } 369 370 static __inline void 371 do_cpuid(u_int ax, u_int *p) 372 { 373 __asm __volatile( 374 "cpuid" 375 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 376 : "0" (ax) 377 ); 378 } 379 380 static __inline u_int64_t 381 rdmsr(u_int msr) 382 { 383 u_int64_t rv; 384 385 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 386 return (rv); 387 } 388 389 static __inline u_int64_t 390 rdpmc(u_int pmc) 391 { 392 u_int64_t rv; 393 394 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 395 return (rv); 396 } 397 398 static __inline u_int64_t 399 rdtsc(void) 400 { 401 u_int64_t rv; 402 403 __asm __volatile("rdtsc" : "=A" (rv)); 404 return (rv); 405 } 406 407 static __inline void 408 wbinvd(void) 409 { 410 __asm __volatile("wbinvd"); 411 } 412 413 static __inline void 414 write_eflags(u_int ef) 415 { 416 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 417 } 418 419 static __inline void 420 wrmsr(u_int msr, u_int64_t newval) 421 { 422 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 423 } 424 425 static __inline u_int 426 rfs(void) 427 { 428 u_int sel; 429 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 430 return (sel); 431 } 432 433 static __inline u_int 434 rgs(void) 435 { 436 u_int sel; 437 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 438 return (sel); 439 } 440 441 static __inline void 442 load_fs(u_int sel) 443 { 444 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 445 } 446 447 static __inline void 448 load_gs(u_int sel) 449 { 450 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 451 } 452 453 static __inline u_int 454 rdr0(void) 455 { 456 u_int data; 457 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 458 return (data); 459 } 460 461 static __inline void 462 load_dr0(u_int sel) 463 { 464 __asm __volatile("movl %0,%%dr0" : : "r" (sel)); 465 } 466 467 static __inline u_int 468 rdr1(void) 469 { 470 u_int data; 471 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 472 return (data); 473 } 474 475 static __inline void 476 load_dr1(u_int sel) 477 { 478 __asm __volatile("movl %0,%%dr1" : : "r" (sel)); 479 } 480 481 static __inline u_int 482 rdr2(void) 483 { 484 u_int data; 485 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 486 return (data); 487 } 488 489 static __inline void 490 load_dr2(u_int sel) 491 { 492 __asm __volatile("movl %0,%%dr2" : : "r" (sel)); 493 } 494 495 static __inline u_int 496 rdr3(void) 497 { 498 u_int data; 499 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 500 return (data); 501 } 502 503 static __inline void 504 load_dr3(u_int sel) 505 { 506 __asm __volatile("movl %0,%%dr3" : : "r" (sel)); 507 } 508 509 static __inline u_int 510 rdr4(void) 511 { 512 u_int data; 513 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 514 return (data); 515 } 516 517 static __inline void 518 load_dr4(u_int sel) 519 { 520 __asm __volatile("movl %0,%%dr4" : : "r" (sel)); 521 } 522 523 static __inline u_int 524 rdr5(void) 525 { 526 u_int data; 527 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 528 return (data); 529 } 530 531 static __inline void 532 load_dr5(u_int sel) 533 { 534 __asm __volatile("movl %0,%%dr5" : : "r" (sel)); 535 } 536 537 static __inline u_int 538 rdr6(void) 539 { 540 u_int data; 541 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 542 return (data); 543 } 544 545 static __inline void 546 load_dr6(u_int sel) 547 { 548 __asm __volatile("movl %0,%%dr6" : : "r" (sel)); 549 } 550 551 static __inline u_int 552 rdr7(void) 553 { 554 u_int data; 555 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 556 return (data); 557 } 558 559 static __inline void 560 load_dr7(u_int sel) 561 { 562 __asm __volatile("movl %0,%%dr7" : : "r" (sel)); 563 } 564 565 static __inline critical_t 566 cpu_critical_enter(void) 567 { 568 critical_t eflags; 569 570 eflags = read_eflags(); 571 disable_intr(); 572 return (eflags); 573 } 574 575 static __inline void 576 cpu_critical_exit(critical_t eflags) 577 { 578 write_eflags(eflags); 579 } 580 581 static __inline register_t 582 intr_disable(void) 583 { 584 register_t eflags; 585 586 eflags = read_eflags(); 587 disable_intr(); 588 return (eflags); 589 } 590 591 static __inline void 592 intr_restore(register_t eflags) 593 { 594 write_eflags(eflags); 595 } 596 597 #else /* !__GNUC__ */ 598 599 int breakpoint(void); 600 u_int bsfl(u_int mask); 601 u_int bsrl(u_int mask); 602 void disable_intr(void); 603 void do_cpuid(u_int ax, u_int *p); 604 void enable_intr(void); 605 u_char inb(u_int port); 606 u_int inl(u_int port); 607 void insb(u_int port, void *addr, size_t cnt); 608 void insl(u_int port, void *addr, size_t cnt); 609 void insw(u_int port, void *addr, size_t cnt); 610 void invd(void); 611 void invlpg(u_int addr); 612 void invltlb(void); 613 u_short inw(u_int port); 614 void outb(u_int port, u_char data); 615 void outl(u_int port, u_int data); 616 void outsb(u_int port, void *addr, size_t cnt); 617 void outsl(u_int port, void *addr, size_t cnt); 618 void outsw(u_int port, void *addr, size_t cnt); 619 void outw(u_int port, u_short data); 620 u_int rcr2(void); 621 u_int64_t rdmsr(u_int msr); 622 u_int64_t rdpmc(u_int pmc); 623 u_int64_t rdtsc(void); 624 u_int read_eflags(void); 625 void wbinvd(void); 626 void write_eflags(u_int ef); 627 void wrmsr(u_int msr, u_int64_t newval); 628 u_int rfs(void); 629 u_int rgs(void); 630 void load_fs(u_int sel); 631 void load_gs(u_int sel); 632 critical_t cpu_critical_enter(void); 633 void cpu_critical_exit(critical_t eflags); 634 635 #endif /* __GNUC__ */ 636 637 void load_cr0(u_int cr0); 638 void load_cr3(u_int cr3); 639 void load_cr4(u_int cr4); 640 void ltr(u_short sel); 641 u_int rcr0(void); 642 u_int rcr3(void); 643 u_int rcr4(void); 644 void reset_dbregs(void); 645 __END_DECLS 646 647 #endif /* !_MACHINE_CPUFUNC_H_ */ 648