1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern void write_eflags(u_int eflags); 53 extern u_int read_eflags(void); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile uint8_t *) (va)) 59 #define readw(va) (*(volatile uint16_t *) (va)) 60 #define readl(va) (*(volatile uint32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90 } 91 92 static __inline void 93 clflush(u_long addr) 94 { 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97 } 98 99 static __inline void 100 clts(void) 101 { 102 103 __asm __volatile("clts"); 104 } 105 106 static __inline void 107 disable_intr(void) 108 { 109 #ifdef XEN 110 xen_cli(); 111 #else 112 __asm __volatile("cli" : : : "memory"); 113 #endif 114 } 115 116 static __inline void 117 do_cpuid(u_int ax, u_int *p) 118 { 119 __asm __volatile("cpuid" 120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 121 : "0" (ax)); 122 } 123 124 static __inline void 125 cpuid_count(u_int ax, u_int cx, u_int *p) 126 { 127 __asm __volatile("cpuid" 128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 129 : "0" (ax), "c" (cx)); 130 } 131 132 static __inline void 133 enable_intr(void) 134 { 135 #ifdef XEN 136 xen_sti(); 137 #else 138 __asm __volatile("sti"); 139 #endif 140 } 141 142 static __inline void 143 cpu_monitor(const void *addr, u_long extensions, u_int hints) 144 { 145 146 __asm __volatile("monitor" 147 : : "a" (addr), "c" (extensions), "d" (hints)); 148 } 149 150 static __inline void 151 cpu_mwait(u_long extensions, u_int hints) 152 { 153 154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 155 } 156 157 static __inline void 158 lfence(void) 159 { 160 161 __asm __volatile("lfence" : : : "memory"); 162 } 163 164 static __inline void 165 mfence(void) 166 { 167 168 __asm __volatile("mfence" : : : "memory"); 169 } 170 171 #ifdef _KERNEL 172 173 #define HAVE_INLINE_FFS 174 175 static __inline int 176 ffs(int mask) 177 { 178 /* 179 * Note that gcc-2's builtin ffs would be used if we didn't declare 180 * this inline or turn off the builtin. The builtin is faster but 181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 182 * versions. 183 */ 184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 185 } 186 187 #define HAVE_INLINE_FFSL 188 189 static __inline int 190 ffsl(long mask) 191 { 192 return (ffs((int)mask)); 193 } 194 195 #define HAVE_INLINE_FLS 196 197 static __inline int 198 fls(int mask) 199 { 200 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 201 } 202 203 #define HAVE_INLINE_FLSL 204 205 static __inline int 206 flsl(long mask) 207 { 208 return (fls((int)mask)); 209 } 210 211 #endif /* _KERNEL */ 212 213 static __inline void 214 halt(void) 215 { 216 __asm __volatile("hlt"); 217 } 218 219 static __inline u_char 220 inb(u_int port) 221 { 222 u_char data; 223 224 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 225 return (data); 226 } 227 228 static __inline u_int 229 inl(u_int port) 230 { 231 u_int data; 232 233 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 234 return (data); 235 } 236 237 static __inline void 238 insb(u_int port, void *addr, size_t count) 239 { 240 __asm __volatile("cld; rep; insb" 241 : "+D" (addr), "+c" (count) 242 : "d" (port) 243 : "memory"); 244 } 245 246 static __inline void 247 insw(u_int port, void *addr, size_t count) 248 { 249 __asm __volatile("cld; rep; insw" 250 : "+D" (addr), "+c" (count) 251 : "d" (port) 252 : "memory"); 253 } 254 255 static __inline void 256 insl(u_int port, void *addr, size_t count) 257 { 258 __asm __volatile("cld; rep; insl" 259 : "+D" (addr), "+c" (count) 260 : "d" (port) 261 : "memory"); 262 } 263 264 static __inline void 265 invd(void) 266 { 267 __asm __volatile("invd"); 268 } 269 270 static __inline u_short 271 inw(u_int port) 272 { 273 u_short data; 274 275 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 276 return (data); 277 } 278 279 static __inline void 280 outb(u_int port, u_char data) 281 { 282 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 283 } 284 285 static __inline void 286 outl(u_int port, u_int data) 287 { 288 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 289 } 290 291 static __inline void 292 outsb(u_int port, const void *addr, size_t count) 293 { 294 __asm __volatile("cld; rep; outsb" 295 : "+S" (addr), "+c" (count) 296 : "d" (port)); 297 } 298 299 static __inline void 300 outsw(u_int port, const void *addr, size_t count) 301 { 302 __asm __volatile("cld; rep; outsw" 303 : "+S" (addr), "+c" (count) 304 : "d" (port)); 305 } 306 307 static __inline void 308 outsl(u_int port, const void *addr, size_t count) 309 { 310 __asm __volatile("cld; rep; outsl" 311 : "+S" (addr), "+c" (count) 312 : "d" (port)); 313 } 314 315 static __inline void 316 outw(u_int port, u_short data) 317 { 318 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 319 } 320 321 static __inline void 322 ia32_pause(void) 323 { 324 __asm __volatile("pause"); 325 } 326 327 static __inline u_int 328 #ifdef XEN 329 _read_eflags(void) 330 #else 331 read_eflags(void) 332 #endif 333 { 334 u_int ef; 335 336 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 337 return (ef); 338 } 339 340 static __inline uint64_t 341 rdmsr(u_int msr) 342 { 343 uint64_t rv; 344 345 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 346 return (rv); 347 } 348 349 static __inline uint64_t 350 rdpmc(u_int pmc) 351 { 352 uint64_t rv; 353 354 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 355 return (rv); 356 } 357 358 static __inline uint64_t 359 rdtsc(void) 360 { 361 uint64_t rv; 362 363 __asm __volatile("rdtsc" : "=A" (rv)); 364 return (rv); 365 } 366 367 static __inline uint32_t 368 rdtsc32(void) 369 { 370 uint32_t rv; 371 372 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 373 return (rv); 374 } 375 376 static __inline void 377 wbinvd(void) 378 { 379 __asm __volatile("wbinvd"); 380 } 381 382 static __inline void 383 #ifdef XEN 384 _write_eflags(u_int ef) 385 #else 386 write_eflags(u_int ef) 387 #endif 388 { 389 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 390 } 391 392 static __inline void 393 wrmsr(u_int msr, uint64_t newval) 394 { 395 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 396 } 397 398 static __inline void 399 load_cr0(u_int data) 400 { 401 402 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 403 } 404 405 static __inline u_int 406 rcr0(void) 407 { 408 u_int data; 409 410 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 411 return (data); 412 } 413 414 static __inline u_int 415 rcr2(void) 416 { 417 u_int data; 418 419 #ifdef XEN 420 return (xen_rcr2()); 421 #endif 422 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 423 return (data); 424 } 425 426 static __inline void 427 load_cr3(u_int data) 428 { 429 #ifdef XEN 430 xen_load_cr3(data); 431 #else 432 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 433 #endif 434 } 435 436 static __inline u_int 437 rcr3(void) 438 { 439 u_int data; 440 441 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 442 return (data); 443 } 444 445 static __inline void 446 load_cr4(u_int data) 447 { 448 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 449 } 450 451 static __inline u_int 452 rcr4(void) 453 { 454 u_int data; 455 456 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 457 return (data); 458 } 459 460 /* 461 * Global TLB flush (except for thise for pages marked PG_G) 462 */ 463 static __inline void 464 invltlb(void) 465 { 466 #ifdef XEN 467 xen_tlb_flush(); 468 #else 469 load_cr3(rcr3()); 470 #endif 471 } 472 473 /* 474 * TLB flush for an individual page (even if it has PG_G). 475 * Only works on 486+ CPUs (i386 does not have PG_G). 476 */ 477 static __inline void 478 invlpg(u_int addr) 479 { 480 481 #ifdef XEN 482 xen_invlpg(addr); 483 #else 484 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 485 #endif 486 } 487 488 static __inline u_short 489 rfs(void) 490 { 491 u_short sel; 492 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 493 return (sel); 494 } 495 496 static __inline uint64_t 497 rgdt(void) 498 { 499 uint64_t gdtr; 500 __asm __volatile("sgdt %0" : "=m" (gdtr)); 501 return (gdtr); 502 } 503 504 static __inline u_short 505 rgs(void) 506 { 507 u_short sel; 508 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 509 return (sel); 510 } 511 512 static __inline uint64_t 513 ridt(void) 514 { 515 uint64_t idtr; 516 __asm __volatile("sidt %0" : "=m" (idtr)); 517 return (idtr); 518 } 519 520 static __inline u_short 521 rldt(void) 522 { 523 u_short ldtr; 524 __asm __volatile("sldt %0" : "=g" (ldtr)); 525 return (ldtr); 526 } 527 528 static __inline u_short 529 rss(void) 530 { 531 u_short sel; 532 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 533 return (sel); 534 } 535 536 static __inline u_short 537 rtr(void) 538 { 539 u_short tr; 540 __asm __volatile("str %0" : "=g" (tr)); 541 return (tr); 542 } 543 544 static __inline void 545 load_fs(u_short sel) 546 { 547 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 548 } 549 550 static __inline void 551 load_gs(u_short sel) 552 { 553 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 554 } 555 556 static __inline void 557 lidt(struct region_descriptor *addr) 558 { 559 __asm __volatile("lidt (%0)" : : "r" (addr)); 560 } 561 562 static __inline void 563 lldt(u_short sel) 564 { 565 __asm __volatile("lldt %0" : : "r" (sel)); 566 } 567 568 static __inline void 569 ltr(u_short sel) 570 { 571 __asm __volatile("ltr %0" : : "r" (sel)); 572 } 573 574 static __inline u_int 575 rdr0(void) 576 { 577 u_int data; 578 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 579 return (data); 580 } 581 582 static __inline void 583 load_dr0(u_int dr0) 584 { 585 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 586 } 587 588 static __inline u_int 589 rdr1(void) 590 { 591 u_int data; 592 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 593 return (data); 594 } 595 596 static __inline void 597 load_dr1(u_int dr1) 598 { 599 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 600 } 601 602 static __inline u_int 603 rdr2(void) 604 { 605 u_int data; 606 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 607 return (data); 608 } 609 610 static __inline void 611 load_dr2(u_int dr2) 612 { 613 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 614 } 615 616 static __inline u_int 617 rdr3(void) 618 { 619 u_int data; 620 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 621 return (data); 622 } 623 624 static __inline void 625 load_dr3(u_int dr3) 626 { 627 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 628 } 629 630 static __inline u_int 631 rdr4(void) 632 { 633 u_int data; 634 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 635 return (data); 636 } 637 638 static __inline void 639 load_dr4(u_int dr4) 640 { 641 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 642 } 643 644 static __inline u_int 645 rdr5(void) 646 { 647 u_int data; 648 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 649 return (data); 650 } 651 652 static __inline void 653 load_dr5(u_int dr5) 654 { 655 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 656 } 657 658 static __inline u_int 659 rdr6(void) 660 { 661 u_int data; 662 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 663 return (data); 664 } 665 666 static __inline void 667 load_dr6(u_int dr6) 668 { 669 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 670 } 671 672 static __inline u_int 673 rdr7(void) 674 { 675 u_int data; 676 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 677 return (data); 678 } 679 680 static __inline void 681 load_dr7(u_int dr7) 682 { 683 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 684 } 685 686 static __inline u_char 687 read_cyrix_reg(u_char reg) 688 { 689 outb(0x22, reg); 690 return inb(0x23); 691 } 692 693 static __inline void 694 write_cyrix_reg(u_char reg, u_char data) 695 { 696 outb(0x22, reg); 697 outb(0x23, data); 698 } 699 700 static __inline register_t 701 intr_disable(void) 702 { 703 register_t eflags; 704 705 eflags = read_eflags(); 706 disable_intr(); 707 return (eflags); 708 } 709 710 static __inline void 711 intr_restore(register_t eflags) 712 { 713 write_eflags(eflags); 714 } 715 716 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 717 718 int breakpoint(void); 719 u_int bsfl(u_int mask); 720 u_int bsrl(u_int mask); 721 void clflush(u_long addr); 722 void clts(void); 723 void cpuid_count(u_int ax, u_int cx, u_int *p); 724 void disable_intr(void); 725 void do_cpuid(u_int ax, u_int *p); 726 void enable_intr(void); 727 void halt(void); 728 void ia32_pause(void); 729 u_char inb(u_int port); 730 u_int inl(u_int port); 731 void insb(u_int port, void *addr, size_t count); 732 void insl(u_int port, void *addr, size_t count); 733 void insw(u_int port, void *addr, size_t count); 734 register_t intr_disable(void); 735 void intr_restore(register_t ef); 736 void invd(void); 737 void invlpg(u_int addr); 738 void invltlb(void); 739 u_short inw(u_int port); 740 void lidt(struct region_descriptor *addr); 741 void lldt(u_short sel); 742 void load_cr0(u_int cr0); 743 void load_cr3(u_int cr3); 744 void load_cr4(u_int cr4); 745 void load_dr0(u_int dr0); 746 void load_dr1(u_int dr1); 747 void load_dr2(u_int dr2); 748 void load_dr3(u_int dr3); 749 void load_dr4(u_int dr4); 750 void load_dr5(u_int dr5); 751 void load_dr6(u_int dr6); 752 void load_dr7(u_int dr7); 753 void load_fs(u_short sel); 754 void load_gs(u_short sel); 755 void ltr(u_short sel); 756 void outb(u_int port, u_char data); 757 void outl(u_int port, u_int data); 758 void outsb(u_int port, const void *addr, size_t count); 759 void outsl(u_int port, const void *addr, size_t count); 760 void outsw(u_int port, const void *addr, size_t count); 761 void outw(u_int port, u_short data); 762 u_int rcr0(void); 763 u_int rcr2(void); 764 u_int rcr3(void); 765 u_int rcr4(void); 766 uint64_t rdmsr(u_int msr); 767 uint64_t rdpmc(u_int pmc); 768 u_int rdr0(void); 769 u_int rdr1(void); 770 u_int rdr2(void); 771 u_int rdr3(void); 772 u_int rdr4(void); 773 u_int rdr5(void); 774 u_int rdr6(void); 775 u_int rdr7(void); 776 uint64_t rdtsc(void); 777 u_char read_cyrix_reg(u_char reg); 778 u_int read_eflags(void); 779 u_int rfs(void); 780 uint64_t rgdt(void); 781 u_int rgs(void); 782 uint64_t ridt(void); 783 u_short rldt(void); 784 u_short rtr(void); 785 void wbinvd(void); 786 void write_cyrix_reg(u_char reg, u_char data); 787 void write_eflags(u_int ef); 788 void wrmsr(u_int msr, uint64_t newval); 789 790 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 791 792 void reset_dbregs(void); 793 794 #ifdef _KERNEL 795 int rdmsr_safe(u_int msr, uint64_t *val); 796 int wrmsr_safe(u_int msr, uint64_t newval); 797 #endif 798 799 #endif /* !_MACHINE_CPUFUNC_H_ */ 800