1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern void write_eflags(u_int eflags); 53 extern u_int read_eflags(void); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile uint8_t *) (va)) 59 #define readw(va) (*(volatile uint16_t *) (va)) 60 #define readl(va) (*(volatile uint32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90 } 91 92 static __inline void 93 clflush(u_long addr) 94 { 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97 } 98 99 static __inline void 100 clts(void) 101 { 102 103 __asm __volatile("clts"); 104 } 105 106 static __inline void 107 disable_intr(void) 108 { 109 #ifdef XEN 110 xen_cli(); 111 #else 112 __asm __volatile("cli" : : : "memory"); 113 #endif 114 } 115 116 static __inline void 117 do_cpuid(u_int ax, u_int *p) 118 { 119 __asm __volatile("cpuid" 120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 121 : "0" (ax)); 122 } 123 124 static __inline void 125 cpuid_count(u_int ax, u_int cx, u_int *p) 126 { 127 __asm __volatile("cpuid" 128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 129 : "0" (ax), "c" (cx)); 130 } 131 132 static __inline void 133 enable_intr(void) 134 { 135 #ifdef XEN 136 xen_sti(); 137 #else 138 __asm __volatile("sti"); 139 #endif 140 } 141 142 static __inline void 143 cpu_monitor(const void *addr, u_long extensions, u_int hints) 144 { 145 146 __asm __volatile("monitor" 147 : : "a" (addr), "c" (extensions), "d" (hints)); 148 } 149 150 static __inline void 151 cpu_mwait(u_long extensions, u_int hints) 152 { 153 154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 155 } 156 157 static __inline void 158 lfence(void) 159 { 160 161 __asm __volatile("lfence" : : : "memory"); 162 } 163 164 static __inline void 165 mfence(void) 166 { 167 168 __asm __volatile("mfence" : : : "memory"); 169 } 170 171 #ifdef _KERNEL 172 173 #define HAVE_INLINE_FFS 174 175 static __inline int 176 ffs(int mask) 177 { 178 /* 179 * Note that gcc-2's builtin ffs would be used if we didn't declare 180 * this inline or turn off the builtin. The builtin is faster but 181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 182 * versions. 183 */ 184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 185 } 186 187 #define HAVE_INLINE_FFSL 188 189 static __inline int 190 ffsl(long mask) 191 { 192 return (ffs((int)mask)); 193 } 194 195 #define HAVE_INLINE_FLS 196 197 static __inline int 198 fls(int mask) 199 { 200 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 201 } 202 203 #define HAVE_INLINE_FLSL 204 205 static __inline int 206 flsl(long mask) 207 { 208 return (fls((int)mask)); 209 } 210 211 #endif /* _KERNEL */ 212 213 static __inline void 214 halt(void) 215 { 216 __asm __volatile("hlt"); 217 } 218 219 static __inline u_char 220 inb(u_int port) 221 { 222 u_char data; 223 224 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 225 return (data); 226 } 227 228 static __inline u_int 229 inl(u_int port) 230 { 231 u_int data; 232 233 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 234 return (data); 235 } 236 237 static __inline void 238 insb(u_int port, void *addr, size_t count) 239 { 240 __asm __volatile("cld; rep; insb" 241 : "+D" (addr), "+c" (count) 242 : "d" (port) 243 : "memory"); 244 } 245 246 static __inline void 247 insw(u_int port, void *addr, size_t count) 248 { 249 __asm __volatile("cld; rep; insw" 250 : "+D" (addr), "+c" (count) 251 : "d" (port) 252 : "memory"); 253 } 254 255 static __inline void 256 insl(u_int port, void *addr, size_t count) 257 { 258 __asm __volatile("cld; rep; insl" 259 : "+D" (addr), "+c" (count) 260 : "d" (port) 261 : "memory"); 262 } 263 264 static __inline void 265 invd(void) 266 { 267 __asm __volatile("invd"); 268 } 269 270 static __inline u_short 271 inw(u_int port) 272 { 273 u_short data; 274 275 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 276 return (data); 277 } 278 279 static __inline void 280 outb(u_int port, u_char data) 281 { 282 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 283 } 284 285 static __inline void 286 outl(u_int port, u_int data) 287 { 288 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 289 } 290 291 static __inline void 292 outsb(u_int port, const void *addr, size_t count) 293 { 294 __asm __volatile("cld; rep; outsb" 295 : "+S" (addr), "+c" (count) 296 : "d" (port)); 297 } 298 299 static __inline void 300 outsw(u_int port, const void *addr, size_t count) 301 { 302 __asm __volatile("cld; rep; outsw" 303 : "+S" (addr), "+c" (count) 304 : "d" (port)); 305 } 306 307 static __inline void 308 outsl(u_int port, const void *addr, size_t count) 309 { 310 __asm __volatile("cld; rep; outsl" 311 : "+S" (addr), "+c" (count) 312 : "d" (port)); 313 } 314 315 static __inline void 316 outw(u_int port, u_short data) 317 { 318 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 319 } 320 321 static __inline void 322 ia32_pause(void) 323 { 324 __asm __volatile("pause"); 325 } 326 327 static __inline u_int 328 #ifdef XEN 329 _read_eflags(void) 330 #else 331 read_eflags(void) 332 #endif 333 { 334 u_int ef; 335 336 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 337 return (ef); 338 } 339 340 static __inline uint64_t 341 rdmsr(u_int msr) 342 { 343 uint64_t rv; 344 345 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 346 return (rv); 347 } 348 349 static __inline uint64_t 350 rdpmc(u_int pmc) 351 { 352 uint64_t rv; 353 354 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 355 return (rv); 356 } 357 358 static __inline uint64_t 359 rdtsc(void) 360 { 361 uint64_t rv; 362 363 __asm __volatile("rdtsc" : "=A" (rv)); 364 return (rv); 365 } 366 367 static __inline uint32_t 368 rdtsc32(void) 369 { 370 uint32_t rv; 371 372 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 373 return (rv); 374 } 375 376 static __inline void 377 wbinvd(void) 378 { 379 __asm __volatile("wbinvd"); 380 } 381 382 static __inline void 383 #ifdef XEN 384 _write_eflags(u_int ef) 385 #else 386 write_eflags(u_int ef) 387 #endif 388 { 389 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 390 } 391 392 static __inline void 393 wrmsr(u_int msr, uint64_t newval) 394 { 395 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 396 } 397 398 static __inline void 399 load_cr0(u_int data) 400 { 401 402 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 403 } 404 405 static __inline u_int 406 rcr0(void) 407 { 408 u_int data; 409 410 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 411 return (data); 412 } 413 414 static __inline u_int 415 rcr2(void) 416 { 417 u_int data; 418 419 #ifdef XEN 420 return (xen_rcr2()); 421 #endif 422 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 423 return (data); 424 } 425 426 static __inline void 427 load_cr3(u_int data) 428 { 429 #ifdef XEN 430 xen_load_cr3(data); 431 #else 432 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 433 #endif 434 } 435 436 static __inline u_int 437 rcr3(void) 438 { 439 u_int data; 440 441 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 442 return (data); 443 } 444 445 static __inline void 446 load_cr4(u_int data) 447 { 448 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 449 } 450 451 static __inline u_int 452 rcr4(void) 453 { 454 u_int data; 455 456 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 457 return (data); 458 } 459 460 static __inline uint64_t 461 rxcr(u_int reg) 462 { 463 u_int low, high; 464 465 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 466 return (low | ((uint64_t)high << 32)); 467 } 468 469 static __inline void 470 load_xcr(u_int reg, uint64_t val) 471 { 472 u_int low, high; 473 474 low = val; 475 high = val >> 32; 476 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 477 } 478 479 /* 480 * Global TLB flush (except for thise for pages marked PG_G) 481 */ 482 static __inline void 483 invltlb(void) 484 { 485 #ifdef XEN 486 xen_tlb_flush(); 487 #else 488 load_cr3(rcr3()); 489 #endif 490 } 491 492 /* 493 * TLB flush for an individual page (even if it has PG_G). 494 * Only works on 486+ CPUs (i386 does not have PG_G). 495 */ 496 static __inline void 497 invlpg(u_int addr) 498 { 499 500 #ifdef XEN 501 xen_invlpg(addr); 502 #else 503 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 504 #endif 505 } 506 507 static __inline u_short 508 rfs(void) 509 { 510 u_short sel; 511 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 512 return (sel); 513 } 514 515 static __inline uint64_t 516 rgdt(void) 517 { 518 uint64_t gdtr; 519 __asm __volatile("sgdt %0" : "=m" (gdtr)); 520 return (gdtr); 521 } 522 523 static __inline u_short 524 rgs(void) 525 { 526 u_short sel; 527 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 528 return (sel); 529 } 530 531 static __inline uint64_t 532 ridt(void) 533 { 534 uint64_t idtr; 535 __asm __volatile("sidt %0" : "=m" (idtr)); 536 return (idtr); 537 } 538 539 static __inline u_short 540 rldt(void) 541 { 542 u_short ldtr; 543 __asm __volatile("sldt %0" : "=g" (ldtr)); 544 return (ldtr); 545 } 546 547 static __inline u_short 548 rss(void) 549 { 550 u_short sel; 551 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 552 return (sel); 553 } 554 555 static __inline u_short 556 rtr(void) 557 { 558 u_short tr; 559 __asm __volatile("str %0" : "=g" (tr)); 560 return (tr); 561 } 562 563 static __inline void 564 load_fs(u_short sel) 565 { 566 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 567 } 568 569 static __inline void 570 load_gs(u_short sel) 571 { 572 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 573 } 574 575 static __inline void 576 lidt(struct region_descriptor *addr) 577 { 578 __asm __volatile("lidt (%0)" : : "r" (addr)); 579 } 580 581 static __inline void 582 lldt(u_short sel) 583 { 584 __asm __volatile("lldt %0" : : "r" (sel)); 585 } 586 587 static __inline void 588 ltr(u_short sel) 589 { 590 __asm __volatile("ltr %0" : : "r" (sel)); 591 } 592 593 static __inline u_int 594 rdr0(void) 595 { 596 u_int data; 597 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 598 return (data); 599 } 600 601 static __inline void 602 load_dr0(u_int dr0) 603 { 604 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 605 } 606 607 static __inline u_int 608 rdr1(void) 609 { 610 u_int data; 611 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 612 return (data); 613 } 614 615 static __inline void 616 load_dr1(u_int dr1) 617 { 618 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 619 } 620 621 static __inline u_int 622 rdr2(void) 623 { 624 u_int data; 625 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 626 return (data); 627 } 628 629 static __inline void 630 load_dr2(u_int dr2) 631 { 632 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 633 } 634 635 static __inline u_int 636 rdr3(void) 637 { 638 u_int data; 639 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 640 return (data); 641 } 642 643 static __inline void 644 load_dr3(u_int dr3) 645 { 646 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 647 } 648 649 static __inline u_int 650 rdr4(void) 651 { 652 u_int data; 653 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 654 return (data); 655 } 656 657 static __inline void 658 load_dr4(u_int dr4) 659 { 660 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 661 } 662 663 static __inline u_int 664 rdr5(void) 665 { 666 u_int data; 667 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 668 return (data); 669 } 670 671 static __inline void 672 load_dr5(u_int dr5) 673 { 674 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 675 } 676 677 static __inline u_int 678 rdr6(void) 679 { 680 u_int data; 681 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 682 return (data); 683 } 684 685 static __inline void 686 load_dr6(u_int dr6) 687 { 688 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 689 } 690 691 static __inline u_int 692 rdr7(void) 693 { 694 u_int data; 695 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 696 return (data); 697 } 698 699 static __inline void 700 load_dr7(u_int dr7) 701 { 702 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 703 } 704 705 static __inline u_char 706 read_cyrix_reg(u_char reg) 707 { 708 outb(0x22, reg); 709 return inb(0x23); 710 } 711 712 static __inline void 713 write_cyrix_reg(u_char reg, u_char data) 714 { 715 outb(0x22, reg); 716 outb(0x23, data); 717 } 718 719 static __inline register_t 720 intr_disable(void) 721 { 722 register_t eflags; 723 724 eflags = read_eflags(); 725 disable_intr(); 726 return (eflags); 727 } 728 729 static __inline void 730 intr_restore(register_t eflags) 731 { 732 write_eflags(eflags); 733 } 734 735 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 736 737 int breakpoint(void); 738 u_int bsfl(u_int mask); 739 u_int bsrl(u_int mask); 740 void clflush(u_long addr); 741 void clts(void); 742 void cpuid_count(u_int ax, u_int cx, u_int *p); 743 void disable_intr(void); 744 void do_cpuid(u_int ax, u_int *p); 745 void enable_intr(void); 746 void halt(void); 747 void ia32_pause(void); 748 u_char inb(u_int port); 749 u_int inl(u_int port); 750 void insb(u_int port, void *addr, size_t count); 751 void insl(u_int port, void *addr, size_t count); 752 void insw(u_int port, void *addr, size_t count); 753 register_t intr_disable(void); 754 void intr_restore(register_t ef); 755 void invd(void); 756 void invlpg(u_int addr); 757 void invltlb(void); 758 u_short inw(u_int port); 759 void lidt(struct region_descriptor *addr); 760 void lldt(u_short sel); 761 void load_cr0(u_int cr0); 762 void load_cr3(u_int cr3); 763 void load_cr4(u_int cr4); 764 void load_dr0(u_int dr0); 765 void load_dr1(u_int dr1); 766 void load_dr2(u_int dr2); 767 void load_dr3(u_int dr3); 768 void load_dr4(u_int dr4); 769 void load_dr5(u_int dr5); 770 void load_dr6(u_int dr6); 771 void load_dr7(u_int dr7); 772 void load_fs(u_short sel); 773 void load_gs(u_short sel); 774 void ltr(u_short sel); 775 void outb(u_int port, u_char data); 776 void outl(u_int port, u_int data); 777 void outsb(u_int port, const void *addr, size_t count); 778 void outsl(u_int port, const void *addr, size_t count); 779 void outsw(u_int port, const void *addr, size_t count); 780 void outw(u_int port, u_short data); 781 u_int rcr0(void); 782 u_int rcr2(void); 783 u_int rcr3(void); 784 u_int rcr4(void); 785 uint64_t rdmsr(u_int msr); 786 uint64_t rdpmc(u_int pmc); 787 u_int rdr0(void); 788 u_int rdr1(void); 789 u_int rdr2(void); 790 u_int rdr3(void); 791 u_int rdr4(void); 792 u_int rdr5(void); 793 u_int rdr6(void); 794 u_int rdr7(void); 795 uint64_t rdtsc(void); 796 u_char read_cyrix_reg(u_char reg); 797 u_int read_eflags(void); 798 u_int rfs(void); 799 uint64_t rgdt(void); 800 u_int rgs(void); 801 uint64_t ridt(void); 802 u_short rldt(void); 803 u_short rtr(void); 804 void wbinvd(void); 805 void write_cyrix_reg(u_char reg, u_char data); 806 void write_eflags(u_int ef); 807 void wrmsr(u_int msr, uint64_t newval); 808 809 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 810 811 void reset_dbregs(void); 812 813 #ifdef _KERNEL 814 int rdmsr_safe(u_int msr, uint64_t *val); 815 int wrmsr_safe(u_int msr, uint64_t newval); 816 #endif 817 818 #endif /* !_MACHINE_CPUFUNC_H_ */ 819