1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 struct region_descriptor; 46 47 #define readb(va) (*(volatile uint8_t *) (va)) 48 #define readw(va) (*(volatile uint16_t *) (va)) 49 #define readl(va) (*(volatile uint32_t *) (va)) 50 51 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 52 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 53 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 54 55 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 56 57 static __inline void 58 breakpoint(void) 59 { 60 __asm __volatile("int $3"); 61 } 62 63 static __inline u_int 64 bsfl(u_int mask) 65 { 66 u_int result; 67 68 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 69 return (result); 70 } 71 72 static __inline u_int 73 bsrl(u_int mask) 74 { 75 u_int result; 76 77 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 78 return (result); 79 } 80 81 static __inline void 82 clflush(u_long addr) 83 { 84 85 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 86 } 87 88 static __inline void 89 clflushopt(u_long addr) 90 { 91 92 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 93 } 94 95 static __inline void 96 clts(void) 97 { 98 99 __asm __volatile("clts"); 100 } 101 102 static __inline void 103 disable_intr(void) 104 { 105 106 __asm __volatile("cli" : : : "memory"); 107 } 108 109 static __inline void 110 do_cpuid(u_int ax, u_int *p) 111 { 112 __asm __volatile("cpuid" 113 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 114 : "0" (ax)); 115 } 116 117 static __inline void 118 cpuid_count(u_int ax, u_int cx, u_int *p) 119 { 120 __asm __volatile("cpuid" 121 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 122 : "0" (ax), "c" (cx)); 123 } 124 125 static __inline void 126 enable_intr(void) 127 { 128 129 __asm __volatile("sti"); 130 } 131 132 static __inline void 133 cpu_monitor(const void *addr, u_long extensions, u_int hints) 134 { 135 136 __asm __volatile("monitor" 137 : : "a" (addr), "c" (extensions), "d" (hints)); 138 } 139 140 static __inline void 141 cpu_mwait(u_long extensions, u_int hints) 142 { 143 144 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 145 } 146 147 static __inline void 148 lfence(void) 149 { 150 151 __asm __volatile("lfence" : : : "memory"); 152 } 153 154 static __inline void 155 mfence(void) 156 { 157 158 __asm __volatile("mfence" : : : "memory"); 159 } 160 161 static __inline void 162 sfence(void) 163 { 164 165 __asm __volatile("sfence" : : : "memory"); 166 } 167 168 #ifdef _KERNEL 169 170 #define HAVE_INLINE_FFS 171 172 static __inline int 173 ffs(int mask) 174 { 175 /* 176 * Note that gcc-2's builtin ffs would be used if we didn't declare 177 * this inline or turn off the builtin. The builtin is faster but 178 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 179 * versions. 180 */ 181 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 182 } 183 184 #define HAVE_INLINE_FFSL 185 186 static __inline int 187 ffsl(long mask) 188 { 189 return (ffs((int)mask)); 190 } 191 192 #define HAVE_INLINE_FLS 193 194 static __inline int 195 fls(int mask) 196 { 197 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 198 } 199 200 #define HAVE_INLINE_FLSL 201 202 static __inline int 203 flsl(long mask) 204 { 205 return (fls((int)mask)); 206 } 207 208 #endif /* _KERNEL */ 209 210 static __inline void 211 halt(void) 212 { 213 __asm __volatile("hlt"); 214 } 215 216 static __inline u_char 217 inb(u_int port) 218 { 219 u_char data; 220 221 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 222 return (data); 223 } 224 225 static __inline u_int 226 inl(u_int port) 227 { 228 u_int data; 229 230 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 231 return (data); 232 } 233 234 static __inline void 235 insb(u_int port, void *addr, size_t count) 236 { 237 __asm __volatile("cld; rep; insb" 238 : "+D" (addr), "+c" (count) 239 : "d" (port) 240 : "memory"); 241 } 242 243 static __inline void 244 insw(u_int port, void *addr, size_t count) 245 { 246 __asm __volatile("cld; rep; insw" 247 : "+D" (addr), "+c" (count) 248 : "d" (port) 249 : "memory"); 250 } 251 252 static __inline void 253 insl(u_int port, void *addr, size_t count) 254 { 255 __asm __volatile("cld; rep; insl" 256 : "+D" (addr), "+c" (count) 257 : "d" (port) 258 : "memory"); 259 } 260 261 static __inline void 262 invd(void) 263 { 264 __asm __volatile("invd"); 265 } 266 267 static __inline u_short 268 inw(u_int port) 269 { 270 u_short data; 271 272 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 273 return (data); 274 } 275 276 static __inline void 277 outb(u_int port, u_char data) 278 { 279 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 280 } 281 282 static __inline void 283 outl(u_int port, u_int data) 284 { 285 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 286 } 287 288 static __inline void 289 outsb(u_int port, const void *addr, size_t count) 290 { 291 __asm __volatile("cld; rep; outsb" 292 : "+S" (addr), "+c" (count) 293 : "d" (port)); 294 } 295 296 static __inline void 297 outsw(u_int port, const void *addr, size_t count) 298 { 299 __asm __volatile("cld; rep; outsw" 300 : "+S" (addr), "+c" (count) 301 : "d" (port)); 302 } 303 304 static __inline void 305 outsl(u_int port, const void *addr, size_t count) 306 { 307 __asm __volatile("cld; rep; outsl" 308 : "+S" (addr), "+c" (count) 309 : "d" (port)); 310 } 311 312 static __inline void 313 outw(u_int port, u_short data) 314 { 315 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 316 } 317 318 static __inline void 319 ia32_pause(void) 320 { 321 __asm __volatile("pause"); 322 } 323 324 static __inline u_int 325 read_eflags(void) 326 { 327 u_int ef; 328 329 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 330 return (ef); 331 } 332 333 static __inline uint64_t 334 rdmsr(u_int msr) 335 { 336 uint64_t rv; 337 338 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 339 return (rv); 340 } 341 342 static __inline uint32_t 343 rdmsr32(u_int msr) 344 { 345 uint32_t low; 346 347 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 348 return (low); 349 } 350 351 static __inline uint64_t 352 rdpmc(u_int pmc) 353 { 354 uint64_t rv; 355 356 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 357 return (rv); 358 } 359 360 static __inline uint64_t 361 rdtsc(void) 362 { 363 uint64_t rv; 364 365 __asm __volatile("rdtsc" : "=A" (rv)); 366 return (rv); 367 } 368 369 static __inline uint32_t 370 rdtsc32(void) 371 { 372 uint32_t rv; 373 374 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 375 return (rv); 376 } 377 378 static __inline void 379 wbinvd(void) 380 { 381 __asm __volatile("wbinvd"); 382 } 383 384 static __inline void 385 write_eflags(u_int ef) 386 { 387 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 388 } 389 390 static __inline void 391 wrmsr(u_int msr, uint64_t newval) 392 { 393 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 394 } 395 396 static __inline void 397 load_cr0(u_int data) 398 { 399 400 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 401 } 402 403 static __inline u_int 404 rcr0(void) 405 { 406 u_int data; 407 408 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 409 return (data); 410 } 411 412 static __inline u_int 413 rcr2(void) 414 { 415 u_int data; 416 417 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 418 return (data); 419 } 420 421 static __inline void 422 load_cr3(u_int data) 423 { 424 425 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 426 } 427 428 static __inline u_int 429 rcr3(void) 430 { 431 u_int data; 432 433 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 434 return (data); 435 } 436 437 static __inline void 438 load_cr4(u_int data) 439 { 440 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 441 } 442 443 static __inline u_int 444 rcr4(void) 445 { 446 u_int data; 447 448 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 449 return (data); 450 } 451 452 static __inline uint64_t 453 rxcr(u_int reg) 454 { 455 u_int low, high; 456 457 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 458 return (low | ((uint64_t)high << 32)); 459 } 460 461 static __inline void 462 load_xcr(u_int reg, uint64_t val) 463 { 464 u_int low, high; 465 466 low = val; 467 high = val >> 32; 468 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 469 } 470 471 /* 472 * Global TLB flush (except for thise for pages marked PG_G) 473 */ 474 static __inline void 475 invltlb(void) 476 { 477 478 load_cr3(rcr3()); 479 } 480 481 /* 482 * TLB flush for an individual page (even if it has PG_G). 483 * Only works on 486+ CPUs (i386 does not have PG_G). 484 */ 485 static __inline void 486 invlpg(u_int addr) 487 { 488 489 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 490 } 491 492 static __inline u_short 493 rfs(void) 494 { 495 u_short sel; 496 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 497 return (sel); 498 } 499 500 static __inline uint64_t 501 rgdt(void) 502 { 503 uint64_t gdtr; 504 __asm __volatile("sgdt %0" : "=m" (gdtr)); 505 return (gdtr); 506 } 507 508 static __inline u_short 509 rgs(void) 510 { 511 u_short sel; 512 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 513 return (sel); 514 } 515 516 static __inline uint64_t 517 ridt(void) 518 { 519 uint64_t idtr; 520 __asm __volatile("sidt %0" : "=m" (idtr)); 521 return (idtr); 522 } 523 524 static __inline u_short 525 rldt(void) 526 { 527 u_short ldtr; 528 __asm __volatile("sldt %0" : "=g" (ldtr)); 529 return (ldtr); 530 } 531 532 static __inline u_short 533 rss(void) 534 { 535 u_short sel; 536 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 537 return (sel); 538 } 539 540 static __inline u_short 541 rtr(void) 542 { 543 u_short tr; 544 __asm __volatile("str %0" : "=g" (tr)); 545 return (tr); 546 } 547 548 static __inline void 549 load_fs(u_short sel) 550 { 551 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 552 } 553 554 static __inline void 555 load_gs(u_short sel) 556 { 557 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 558 } 559 560 static __inline void 561 lidt(struct region_descriptor *addr) 562 { 563 __asm __volatile("lidt (%0)" : : "r" (addr)); 564 } 565 566 static __inline void 567 lldt(u_short sel) 568 { 569 __asm __volatile("lldt %0" : : "r" (sel)); 570 } 571 572 static __inline void 573 ltr(u_short sel) 574 { 575 __asm __volatile("ltr %0" : : "r" (sel)); 576 } 577 578 static __inline u_int 579 rdr0(void) 580 { 581 u_int data; 582 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 583 return (data); 584 } 585 586 static __inline void 587 load_dr0(u_int dr0) 588 { 589 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 590 } 591 592 static __inline u_int 593 rdr1(void) 594 { 595 u_int data; 596 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 597 return (data); 598 } 599 600 static __inline void 601 load_dr1(u_int dr1) 602 { 603 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 604 } 605 606 static __inline u_int 607 rdr2(void) 608 { 609 u_int data; 610 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 611 return (data); 612 } 613 614 static __inline void 615 load_dr2(u_int dr2) 616 { 617 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 618 } 619 620 static __inline u_int 621 rdr3(void) 622 { 623 u_int data; 624 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 625 return (data); 626 } 627 628 static __inline void 629 load_dr3(u_int dr3) 630 { 631 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 632 } 633 634 static __inline u_int 635 rdr4(void) 636 { 637 u_int data; 638 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 639 return (data); 640 } 641 642 static __inline void 643 load_dr4(u_int dr4) 644 { 645 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 646 } 647 648 static __inline u_int 649 rdr5(void) 650 { 651 u_int data; 652 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 653 return (data); 654 } 655 656 static __inline void 657 load_dr5(u_int dr5) 658 { 659 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 660 } 661 662 static __inline u_int 663 rdr6(void) 664 { 665 u_int data; 666 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 667 return (data); 668 } 669 670 static __inline void 671 load_dr6(u_int dr6) 672 { 673 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 674 } 675 676 static __inline u_int 677 rdr7(void) 678 { 679 u_int data; 680 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 681 return (data); 682 } 683 684 static __inline void 685 load_dr7(u_int dr7) 686 { 687 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 688 } 689 690 static __inline u_char 691 read_cyrix_reg(u_char reg) 692 { 693 outb(0x22, reg); 694 return inb(0x23); 695 } 696 697 static __inline void 698 write_cyrix_reg(u_char reg, u_char data) 699 { 700 outb(0x22, reg); 701 outb(0x23, data); 702 } 703 704 static __inline register_t 705 intr_disable(void) 706 { 707 register_t eflags; 708 709 eflags = read_eflags(); 710 disable_intr(); 711 return (eflags); 712 } 713 714 static __inline void 715 intr_restore(register_t eflags) 716 { 717 write_eflags(eflags); 718 } 719 720 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 721 722 int breakpoint(void); 723 u_int bsfl(u_int mask); 724 u_int bsrl(u_int mask); 725 void clflush(u_long addr); 726 void clts(void); 727 void cpuid_count(u_int ax, u_int cx, u_int *p); 728 void disable_intr(void); 729 void do_cpuid(u_int ax, u_int *p); 730 void enable_intr(void); 731 void halt(void); 732 void ia32_pause(void); 733 u_char inb(u_int port); 734 u_int inl(u_int port); 735 void insb(u_int port, void *addr, size_t count); 736 void insl(u_int port, void *addr, size_t count); 737 void insw(u_int port, void *addr, size_t count); 738 register_t intr_disable(void); 739 void intr_restore(register_t ef); 740 void invd(void); 741 void invlpg(u_int addr); 742 void invltlb(void); 743 u_short inw(u_int port); 744 void lidt(struct region_descriptor *addr); 745 void lldt(u_short sel); 746 void load_cr0(u_int cr0); 747 void load_cr3(u_int cr3); 748 void load_cr4(u_int cr4); 749 void load_dr0(u_int dr0); 750 void load_dr1(u_int dr1); 751 void load_dr2(u_int dr2); 752 void load_dr3(u_int dr3); 753 void load_dr4(u_int dr4); 754 void load_dr5(u_int dr5); 755 void load_dr6(u_int dr6); 756 void load_dr7(u_int dr7); 757 void load_fs(u_short sel); 758 void load_gs(u_short sel); 759 void ltr(u_short sel); 760 void outb(u_int port, u_char data); 761 void outl(u_int port, u_int data); 762 void outsb(u_int port, const void *addr, size_t count); 763 void outsl(u_int port, const void *addr, size_t count); 764 void outsw(u_int port, const void *addr, size_t count); 765 void outw(u_int port, u_short data); 766 u_int rcr0(void); 767 u_int rcr2(void); 768 u_int rcr3(void); 769 u_int rcr4(void); 770 uint64_t rdmsr(u_int msr); 771 uint64_t rdpmc(u_int pmc); 772 u_int rdr0(void); 773 u_int rdr1(void); 774 u_int rdr2(void); 775 u_int rdr3(void); 776 u_int rdr4(void); 777 u_int rdr5(void); 778 u_int rdr6(void); 779 u_int rdr7(void); 780 uint64_t rdtsc(void); 781 u_char read_cyrix_reg(u_char reg); 782 u_int read_eflags(void); 783 u_int rfs(void); 784 uint64_t rgdt(void); 785 u_int rgs(void); 786 uint64_t ridt(void); 787 u_short rldt(void); 788 u_short rtr(void); 789 void wbinvd(void); 790 void write_cyrix_reg(u_char reg, u_char data); 791 void write_eflags(u_int ef); 792 void wrmsr(u_int msr, uint64_t newval); 793 794 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 795 796 void reset_dbregs(void); 797 798 #ifdef _KERNEL 799 int rdmsr_safe(u_int msr, uint64_t *val); 800 int wrmsr_safe(u_int msr, uint64_t newval); 801 #endif 802 803 #endif /* !_MACHINE_CPUFUNC_H_ */ 804