1 /*- 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1993 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 33 /* 34 * Functions to provide access to special i386 instructions. 35 * This in included in sys/systm.h, and that file should be 36 * used in preference to this. 37 */ 38 39 #ifndef _MACHINE_CPUFUNC_H_ 40 #define _MACHINE_CPUFUNC_H_ 41 42 #ifndef _SYS_CDEFS_H_ 43 #error this file needs sys/cdefs.h as a prerequisite 44 #endif 45 46 struct region_descriptor; 47 48 #define readb(va) (*(volatile u_int8_t *) (va)) 49 #define readw(va) (*(volatile u_int16_t *) (va)) 50 #define readl(va) (*(volatile u_int32_t *) (va)) 51 #define readq(va) (*(volatile u_int64_t *) (va)) 52 53 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 54 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 55 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 56 #define writeq(va, d) (*(volatile u_int64_t *) (va) = (d)) 57 58 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 59 60 static __inline void 61 breakpoint(void) 62 { 63 __asm __volatile("int $3"); 64 } 65 66 static __inline u_int 67 bsfl(u_int mask) 68 { 69 u_int result; 70 71 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 72 return (result); 73 } 74 75 static __inline u_long 76 bsfq(u_long mask) 77 { 78 u_long result; 79 80 __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask)); 81 return (result); 82 } 83 84 static __inline u_int 85 bsrl(u_int mask) 86 { 87 u_int result; 88 89 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 90 return (result); 91 } 92 93 static __inline u_long 94 bsrq(u_long mask) 95 { 96 u_long result; 97 98 __asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask)); 99 return (result); 100 } 101 102 static __inline void 103 disable_intr(void) 104 { 105 __asm __volatile("cli" : : : "memory"); 106 } 107 108 static __inline void 109 do_cpuid(u_int ax, u_int *p) 110 { 111 __asm __volatile("cpuid" 112 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 113 : "0" (ax)); 114 } 115 116 static __inline void 117 enable_intr(void) 118 { 119 __asm __volatile("sti"); 120 } 121 122 #ifdef _KERNEL 123 124 #define HAVE_INLINE_FFS 125 #define ffs(x) __builtin_ffs(x) 126 127 #define HAVE_INLINE_FFSL 128 129 static __inline int 130 ffsl(long mask) 131 { 132 return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1); 133 } 134 135 #define HAVE_INLINE_FLS 136 137 static __inline int 138 fls(int mask) 139 { 140 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 141 } 142 143 #define HAVE_INLINE_FLSL 144 145 static __inline int 146 flsl(long mask) 147 { 148 return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1); 149 } 150 151 #endif /* _KERNEL */ 152 153 static __inline void 154 halt(void) 155 { 156 __asm __volatile("hlt"); 157 } 158 159 #if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3 160 161 #define inb(port) inbv(port) 162 #define outb(port, data) outbv(port, data) 163 164 #else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */ 165 166 /* 167 * The following complications are to get around gcc not having a 168 * constraint letter for the range 0..255. We still put "d" in the 169 * constraint because "i" isn't a valid constraint when the port 170 * isn't constant. This only matters for -O0 because otherwise 171 * the non-working version gets optimized away. 172 * 173 * Use an expression-statement instead of a conditional expression 174 * because gcc-2.6.0 would promote the operands of the conditional 175 * and produce poor code for "if ((inb(var) & const1) == const2)". 176 * 177 * The unnecessary test `(port) < 0x10000' is to generate a warning if 178 * the `port' has type u_short or smaller. Such types are pessimal. 179 * This actually only works for signed types. The range check is 180 * careful to avoid generating warnings. 181 */ 182 #define inb(port) __extension__ ({ \ 183 u_char _data; \ 184 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 185 && (port) < 0x10000) \ 186 _data = inbc(port); \ 187 else \ 188 _data = inbv(port); \ 189 _data; }) 190 191 #define outb(port, data) ( \ 192 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 193 && (port) < 0x10000 \ 194 ? outbc(port, data) : outbv(port, data)) 195 196 static __inline u_char 197 inbc(u_int port) 198 { 199 u_char data; 200 201 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 202 return (data); 203 } 204 205 static __inline void 206 outbc(u_int port, u_char data) 207 { 208 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 209 } 210 211 #endif /* __GNUCLIKE_BUILTIN_CONSTANT_P */ 212 213 static __inline u_char 214 inbv(u_int port) 215 { 216 u_char data; 217 /* 218 * We use %%dx and not %1 here because i/o is done at %dx and not at 219 * %edx, while gcc generates inferior code (movw instead of movl) 220 * if we tell it to load (u_short) port. 221 */ 222 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 223 return (data); 224 } 225 226 static __inline u_int 227 inl(u_int port) 228 { 229 u_int data; 230 231 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 232 return (data); 233 } 234 235 static __inline void 236 insb(u_int port, void *addr, size_t cnt) 237 { 238 __asm __volatile("cld; rep; insb" 239 : "+D" (addr), "+c" (cnt) 240 : "d" (port) 241 : "memory"); 242 } 243 244 static __inline void 245 insw(u_int port, void *addr, size_t cnt) 246 { 247 __asm __volatile("cld; rep; insw" 248 : "+D" (addr), "+c" (cnt) 249 : "d" (port) 250 : "memory"); 251 } 252 253 static __inline void 254 insl(u_int port, void *addr, size_t cnt) 255 { 256 __asm __volatile("cld; rep; insl" 257 : "+D" (addr), "+c" (cnt) 258 : "d" (port) 259 : "memory"); 260 } 261 262 static __inline void 263 invd(void) 264 { 265 __asm __volatile("invd"); 266 } 267 268 static __inline u_short 269 inw(u_int port) 270 { 271 u_short data; 272 273 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 274 return (data); 275 } 276 277 static __inline void 278 outbv(u_int port, u_char data) 279 { 280 u_char al; 281 /* 282 * Use an unnecessary assignment to help gcc's register allocator. 283 * This make a large difference for gcc-1.40 and a tiny difference 284 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 285 * best results. gcc-2.6.0 can't handle this. 286 */ 287 al = data; 288 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 289 } 290 291 static __inline void 292 outl(u_int port, u_int data) 293 { 294 /* 295 * outl() and outw() aren't used much so we haven't looked at 296 * possible micro-optimizations such as the unnecessary 297 * assignment for them. 298 */ 299 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 300 } 301 302 static __inline void 303 outsb(u_int port, const void *addr, size_t cnt) 304 { 305 __asm __volatile("cld; rep; outsb" 306 : "+S" (addr), "+c" (cnt) 307 : "d" (port)); 308 } 309 310 static __inline void 311 outsw(u_int port, const void *addr, size_t cnt) 312 { 313 __asm __volatile("cld; rep; outsw" 314 : "+S" (addr), "+c" (cnt) 315 : "d" (port)); 316 } 317 318 static __inline void 319 outsl(u_int port, const void *addr, size_t cnt) 320 { 321 __asm __volatile("cld; rep; outsl" 322 : "+S" (addr), "+c" (cnt) 323 : "d" (port)); 324 } 325 326 static __inline void 327 outw(u_int port, u_short data) 328 { 329 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 330 } 331 332 static __inline void 333 ia32_pause(void) 334 { 335 __asm __volatile("pause"); 336 } 337 338 static __inline u_long 339 read_rflags(void) 340 { 341 u_long rf; 342 343 __asm __volatile("pushfq; popq %0" : "=r" (rf)); 344 return (rf); 345 } 346 347 static __inline u_int64_t 348 rdmsr(u_int msr) 349 { 350 u_int32_t low, high; 351 352 __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr)); 353 return (low | ((u_int64_t)high << 32)); 354 } 355 356 static __inline u_int64_t 357 rdpmc(u_int pmc) 358 { 359 u_int32_t low, high; 360 361 __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc)); 362 return (low | ((u_int64_t)high << 32)); 363 } 364 365 static __inline u_int64_t 366 rdtsc(void) 367 { 368 u_int32_t low, high; 369 370 __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 371 return (low | ((u_int64_t)high << 32)); 372 } 373 374 static __inline void 375 wbinvd(void) 376 { 377 __asm __volatile("wbinvd"); 378 } 379 380 static __inline void 381 write_rflags(u_long rf) 382 { 383 __asm __volatile("pushq %0; popfq" : : "r" (rf)); 384 } 385 386 static __inline void 387 wrmsr(u_int msr, u_int64_t newval) 388 { 389 u_int32_t low, high; 390 391 low = newval; 392 high = newval >> 32; 393 __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr)); 394 } 395 396 static __inline void 397 load_cr0(u_long data) 398 { 399 400 __asm __volatile("movq %0,%%cr0" : : "r" (data)); 401 } 402 403 static __inline u_long 404 rcr0(void) 405 { 406 u_long data; 407 408 __asm __volatile("movq %%cr0,%0" : "=r" (data)); 409 return (data); 410 } 411 412 static __inline u_long 413 rcr2(void) 414 { 415 u_long data; 416 417 __asm __volatile("movq %%cr2,%0" : "=r" (data)); 418 return (data); 419 } 420 421 static __inline void 422 load_cr3(u_long data) 423 { 424 425 __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory"); 426 } 427 428 static __inline u_long 429 rcr3(void) 430 { 431 u_long data; 432 433 __asm __volatile("movq %%cr3,%0" : "=r" (data)); 434 return (data); 435 } 436 437 static __inline void 438 load_cr4(u_long data) 439 { 440 __asm __volatile("movq %0,%%cr4" : : "r" (data)); 441 } 442 443 static __inline u_long 444 rcr4(void) 445 { 446 u_long data; 447 448 __asm __volatile("movq %%cr4,%0" : "=r" (data)); 449 return (data); 450 } 451 452 /* 453 * Global TLB flush (except for thise for pages marked PG_G) 454 */ 455 static __inline void 456 invltlb(void) 457 { 458 459 load_cr3(rcr3()); 460 } 461 462 /* 463 * TLB flush for an individual page (even if it has PG_G). 464 * Only works on 486+ CPUs (i386 does not have PG_G). 465 */ 466 static __inline void 467 invlpg(u_long addr) 468 { 469 470 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 471 } 472 473 static __inline u_int 474 rfs(void) 475 { 476 u_int sel; 477 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 478 return (sel); 479 } 480 481 static __inline u_int 482 rgs(void) 483 { 484 u_int sel; 485 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 486 return (sel); 487 } 488 489 static __inline u_int 490 rss(void) 491 { 492 u_int sel; 493 __asm __volatile("movl %%ss,%0" : "=rm" (sel)); 494 return (sel); 495 } 496 497 static __inline void 498 load_ds(u_int sel) 499 { 500 __asm __volatile("movl %0,%%ds" : : "rm" (sel)); 501 } 502 503 static __inline void 504 load_es(u_int sel) 505 { 506 __asm __volatile("movl %0,%%es" : : "rm" (sel)); 507 } 508 509 #ifdef _KERNEL 510 /* This is defined in <machine/specialreg.h> but is too painful to get to */ 511 #ifndef MSR_FSBASE 512 #define MSR_FSBASE 0xc0000100 513 #endif 514 static __inline void 515 load_fs(u_int sel) 516 { 517 register u_int32_t fsbase __asm("ecx"); 518 519 /* Preserve the fsbase value across the selector load */ 520 fsbase = MSR_FSBASE; 521 __asm __volatile("rdmsr; movl %0,%%fs; wrmsr" 522 : : "rm" (sel), "c" (fsbase) : "eax", "edx"); 523 } 524 525 #ifndef MSR_GSBASE 526 #define MSR_GSBASE 0xc0000101 527 #endif 528 static __inline void 529 load_gs(u_int sel) 530 { 531 register u_int32_t gsbase __asm("ecx"); 532 533 /* 534 * Preserve the gsbase value across the selector load. 535 * Note that we have to disable interrupts because the gsbase 536 * being trashed happens to be the kernel gsbase at the time. 537 */ 538 gsbase = MSR_GSBASE; 539 __asm __volatile("pushfq; cli; rdmsr; movl %0,%%gs; wrmsr; popfq" 540 : : "rm" (sel), "c" (gsbase) : "eax", "edx"); 541 } 542 #else 543 /* Usable by userland */ 544 static __inline void 545 load_fs(u_int sel) 546 { 547 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 548 } 549 550 static __inline void 551 load_gs(u_int sel) 552 { 553 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 554 } 555 #endif 556 557 static __inline void 558 lidt(struct region_descriptor *addr) 559 { 560 __asm __volatile("lidt (%0)" : : "r" (addr)); 561 } 562 563 static __inline void 564 lldt(u_short sel) 565 { 566 __asm __volatile("lldt %0" : : "r" (sel)); 567 } 568 569 static __inline void 570 ltr(u_short sel) 571 { 572 __asm __volatile("ltr %0" : : "r" (sel)); 573 } 574 575 static __inline u_int64_t 576 rdr0(void) 577 { 578 u_int64_t data; 579 __asm __volatile("movq %%dr0,%0" : "=r" (data)); 580 return (data); 581 } 582 583 static __inline void 584 load_dr0(u_int64_t dr0) 585 { 586 __asm __volatile("movq %0,%%dr0" : : "r" (dr0)); 587 } 588 589 static __inline u_int64_t 590 rdr1(void) 591 { 592 u_int64_t data; 593 __asm __volatile("movq %%dr1,%0" : "=r" (data)); 594 return (data); 595 } 596 597 static __inline void 598 load_dr1(u_int64_t dr1) 599 { 600 __asm __volatile("movq %0,%%dr1" : : "r" (dr1)); 601 } 602 603 static __inline u_int64_t 604 rdr2(void) 605 { 606 u_int64_t data; 607 __asm __volatile("movq %%dr2,%0" : "=r" (data)); 608 return (data); 609 } 610 611 static __inline void 612 load_dr2(u_int64_t dr2) 613 { 614 __asm __volatile("movq %0,%%dr2" : : "r" (dr2)); 615 } 616 617 static __inline u_int64_t 618 rdr3(void) 619 { 620 u_int64_t data; 621 __asm __volatile("movq %%dr3,%0" : "=r" (data)); 622 return (data); 623 } 624 625 static __inline void 626 load_dr3(u_int64_t dr3) 627 { 628 __asm __volatile("movq %0,%%dr3" : : "r" (dr3)); 629 } 630 631 static __inline u_int64_t 632 rdr4(void) 633 { 634 u_int64_t data; 635 __asm __volatile("movq %%dr4,%0" : "=r" (data)); 636 return (data); 637 } 638 639 static __inline void 640 load_dr4(u_int64_t dr4) 641 { 642 __asm __volatile("movq %0,%%dr4" : : "r" (dr4)); 643 } 644 645 static __inline u_int64_t 646 rdr5(void) 647 { 648 u_int64_t data; 649 __asm __volatile("movq %%dr5,%0" : "=r" (data)); 650 return (data); 651 } 652 653 static __inline void 654 load_dr5(u_int64_t dr5) 655 { 656 __asm __volatile("movq %0,%%dr5" : : "r" (dr5)); 657 } 658 659 static __inline u_int64_t 660 rdr6(void) 661 { 662 u_int64_t data; 663 __asm __volatile("movq %%dr6,%0" : "=r" (data)); 664 return (data); 665 } 666 667 static __inline void 668 load_dr6(u_int64_t dr6) 669 { 670 __asm __volatile("movq %0,%%dr6" : : "r" (dr6)); 671 } 672 673 static __inline u_int64_t 674 rdr7(void) 675 { 676 u_int64_t data; 677 __asm __volatile("movq %%dr7,%0" : "=r" (data)); 678 return (data); 679 } 680 681 static __inline void 682 load_dr7(u_int64_t dr7) 683 { 684 __asm __volatile("movq %0,%%dr7" : : "r" (dr7)); 685 } 686 687 static __inline register_t 688 intr_disable(void) 689 { 690 register_t rflags; 691 692 rflags = read_rflags(); 693 disable_intr(); 694 return (rflags); 695 } 696 697 static __inline void 698 intr_restore(register_t rflags) 699 { 700 write_rflags(rflags); 701 } 702 703 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 704 705 int breakpoint(void); 706 u_int bsfl(u_int mask); 707 u_int bsrl(u_int mask); 708 void disable_intr(void); 709 void do_cpuid(u_int ax, u_int *p); 710 void enable_intr(void); 711 void halt(void); 712 void ia32_pause(void); 713 u_char inb(u_int port); 714 u_int inl(u_int port); 715 void insb(u_int port, void *addr, size_t cnt); 716 void insl(u_int port, void *addr, size_t cnt); 717 void insw(u_int port, void *addr, size_t cnt); 718 register_t intr_disable(void); 719 void intr_restore(register_t rf); 720 void invd(void); 721 void invlpg(u_int addr); 722 void invltlb(void); 723 u_short inw(u_int port); 724 void lidt(struct region_descriptor *addr); 725 void lldt(u_short sel); 726 void load_cr0(u_long cr0); 727 void load_cr3(u_long cr3); 728 void load_cr4(u_long cr4); 729 void load_dr0(u_int64_t dr0); 730 void load_dr1(u_int64_t dr1); 731 void load_dr2(u_int64_t dr2); 732 void load_dr3(u_int64_t dr3); 733 void load_dr4(u_int64_t dr4); 734 void load_dr5(u_int64_t dr5); 735 void load_dr6(u_int64_t dr6); 736 void load_dr7(u_int64_t dr7); 737 void load_fs(u_int sel); 738 void load_gs(u_int sel); 739 void ltr(u_short sel); 740 void outb(u_int port, u_char data); 741 void outl(u_int port, u_int data); 742 void outsb(u_int port, const void *addr, size_t cnt); 743 void outsl(u_int port, const void *addr, size_t cnt); 744 void outsw(u_int port, const void *addr, size_t cnt); 745 void outw(u_int port, u_short data); 746 u_long rcr0(void); 747 u_long rcr2(void); 748 u_long rcr3(void); 749 u_long rcr4(void); 750 u_int64_t rdmsr(u_int msr); 751 u_int64_t rdpmc(u_int pmc); 752 u_int64_t rdr0(void); 753 u_int64_t rdr1(void); 754 u_int64_t rdr2(void); 755 u_int64_t rdr3(void); 756 u_int64_t rdr4(void); 757 u_int64_t rdr5(void); 758 u_int64_t rdr6(void); 759 u_int64_t rdr7(void); 760 u_int64_t rdtsc(void); 761 u_int read_rflags(void); 762 u_int rfs(void); 763 u_int rgs(void); 764 void wbinvd(void); 765 void write_rflags(u_int rf); 766 void wrmsr(u_int msr, u_int64_t newval); 767 768 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 769 770 void reset_dbregs(void); 771 772 #endif /* !_MACHINE_CPUFUNC_H_ */ 773