1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 struct region_descriptor; 46 47 #define readb(va) (*(volatile uint8_t *) (va)) 48 #define readw(va) (*(volatile uint16_t *) (va)) 49 #define readl(va) (*(volatile uint32_t *) (va)) 50 51 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 52 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 53 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 54 55 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 56 57 static __inline void 58 breakpoint(void) 59 { 60 __asm __volatile("int $3"); 61 } 62 63 static __inline u_int 64 bsfl(u_int mask) 65 { 66 u_int result; 67 68 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 69 return (result); 70 } 71 72 static __inline u_int 73 bsrl(u_int mask) 74 { 75 u_int result; 76 77 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 78 return (result); 79 } 80 81 static __inline void 82 clflush(u_long addr) 83 { 84 85 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 86 } 87 88 static __inline void 89 clflushopt(u_long addr) 90 { 91 92 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 93 } 94 95 static __inline void 96 clts(void) 97 { 98 99 __asm __volatile("clts"); 100 } 101 102 static __inline void 103 disable_intr(void) 104 { 105 106 __asm __volatile("cli" : : : "memory"); 107 } 108 109 static __inline void 110 do_cpuid(u_int ax, u_int *p) 111 { 112 __asm __volatile("pushl %%ebx \n\t" 113 "cpuid \n\t" 114 "movl %%ebx, %1 \n\t" 115 "popl %%ebx \n\t" 116 : "=a" (p[0]), "=m" (p[1]), "=c" (p[2]), "=d" (p[3]) 117 : "0" (ax)); 118 } 119 120 static __inline void 121 cpuid_count(u_int ax, u_int cx, u_int *p) 122 { 123 __asm __volatile("cpuid" 124 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 125 : "0" (ax), "c" (cx)); 126 } 127 128 static __inline void 129 enable_intr(void) 130 { 131 132 __asm __volatile("sti"); 133 } 134 135 static __inline void 136 cpu_monitor(const void *addr, u_long extensions, u_int hints) 137 { 138 139 __asm __volatile("monitor" 140 : : "a" (addr), "c" (extensions), "d" (hints)); 141 } 142 143 static __inline void 144 cpu_mwait(u_long extensions, u_int hints) 145 { 146 147 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 148 } 149 150 static __inline void 151 lfence(void) 152 { 153 154 __asm __volatile("lfence" : : : "memory"); 155 } 156 157 static __inline void 158 mfence(void) 159 { 160 161 __asm __volatile("mfence" : : : "memory"); 162 } 163 164 #ifdef _KERNEL 165 166 #define HAVE_INLINE_FFS 167 168 static __inline int 169 ffs(int mask) 170 { 171 /* 172 * Note that gcc-2's builtin ffs would be used if we didn't declare 173 * this inline or turn off the builtin. The builtin is faster but 174 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 175 * versions. 176 */ 177 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 178 } 179 180 #define HAVE_INLINE_FFSL 181 182 static __inline int 183 ffsl(long mask) 184 { 185 return (ffs((int)mask)); 186 } 187 188 #define HAVE_INLINE_FLS 189 190 static __inline int 191 fls(int mask) 192 { 193 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 194 } 195 196 #define HAVE_INLINE_FLSL 197 198 static __inline int 199 flsl(long mask) 200 { 201 return (fls((int)mask)); 202 } 203 204 #endif /* _KERNEL */ 205 206 static __inline void 207 halt(void) 208 { 209 __asm __volatile("hlt"); 210 } 211 212 static __inline u_char 213 inb(u_int port) 214 { 215 u_char data; 216 217 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 218 return (data); 219 } 220 221 static __inline u_int 222 inl(u_int port) 223 { 224 u_int data; 225 226 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 227 return (data); 228 } 229 230 static __inline void 231 insb(u_int port, void *addr, size_t count) 232 { 233 __asm __volatile("cld; rep; insb" 234 : "+D" (addr), "+c" (count) 235 : "d" (port) 236 : "memory"); 237 } 238 239 static __inline void 240 insw(u_int port, void *addr, size_t count) 241 { 242 __asm __volatile("cld; rep; insw" 243 : "+D" (addr), "+c" (count) 244 : "d" (port) 245 : "memory"); 246 } 247 248 static __inline void 249 insl(u_int port, void *addr, size_t count) 250 { 251 __asm __volatile("cld; rep; insl" 252 : "+D" (addr), "+c" (count) 253 : "d" (port) 254 : "memory"); 255 } 256 257 static __inline void 258 invd(void) 259 { 260 __asm __volatile("invd"); 261 } 262 263 static __inline u_short 264 inw(u_int port) 265 { 266 u_short data; 267 268 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 269 return (data); 270 } 271 272 static __inline void 273 outb(u_int port, u_char data) 274 { 275 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 276 } 277 278 static __inline void 279 outl(u_int port, u_int data) 280 { 281 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 282 } 283 284 static __inline void 285 outsb(u_int port, const void *addr, size_t count) 286 { 287 __asm __volatile("cld; rep; outsb" 288 : "+S" (addr), "+c" (count) 289 : "d" (port)); 290 } 291 292 static __inline void 293 outsw(u_int port, const void *addr, size_t count) 294 { 295 __asm __volatile("cld; rep; outsw" 296 : "+S" (addr), "+c" (count) 297 : "d" (port)); 298 } 299 300 static __inline void 301 outsl(u_int port, const void *addr, size_t count) 302 { 303 __asm __volatile("cld; rep; outsl" 304 : "+S" (addr), "+c" (count) 305 : "d" (port)); 306 } 307 308 static __inline void 309 outw(u_int port, u_short data) 310 { 311 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 312 } 313 314 static __inline void 315 ia32_pause(void) 316 { 317 __asm __volatile("pause"); 318 } 319 320 static __inline u_int 321 read_eflags(void) 322 { 323 u_int ef; 324 325 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 326 return (ef); 327 } 328 329 static __inline uint64_t 330 rdmsr(u_int msr) 331 { 332 uint64_t rv; 333 334 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 335 return (rv); 336 } 337 338 static __inline uint32_t 339 rdmsr32(u_int msr) 340 { 341 uint32_t low; 342 343 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 344 return (low); 345 } 346 347 static __inline uint64_t 348 rdpmc(u_int pmc) 349 { 350 uint64_t rv; 351 352 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 353 return (rv); 354 } 355 356 static __inline uint64_t 357 rdtsc(void) 358 { 359 uint64_t rv; 360 361 __asm __volatile("rdtsc" : "=A" (rv)); 362 return (rv); 363 } 364 365 static __inline uint32_t 366 rdtsc32(void) 367 { 368 uint32_t rv; 369 370 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 371 return (rv); 372 } 373 374 static __inline void 375 wbinvd(void) 376 { 377 __asm __volatile("wbinvd"); 378 } 379 380 static __inline void 381 write_eflags(u_int ef) 382 { 383 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 384 } 385 386 static __inline void 387 wrmsr(u_int msr, uint64_t newval) 388 { 389 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 390 } 391 392 static __inline void 393 load_cr0(u_int data) 394 { 395 396 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 397 } 398 399 static __inline u_int 400 rcr0(void) 401 { 402 u_int data; 403 404 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 405 return (data); 406 } 407 408 static __inline u_int 409 rcr2(void) 410 { 411 u_int data; 412 413 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 414 return (data); 415 } 416 417 static __inline void 418 load_cr3(u_int data) 419 { 420 421 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 422 } 423 424 static __inline u_int 425 rcr3(void) 426 { 427 u_int data; 428 429 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 430 return (data); 431 } 432 433 static __inline void 434 load_cr4(u_int data) 435 { 436 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 437 } 438 439 static __inline u_int 440 rcr4(void) 441 { 442 u_int data; 443 444 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 445 return (data); 446 } 447 448 static __inline uint64_t 449 rxcr(u_int reg) 450 { 451 u_int low, high; 452 453 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 454 return (low | ((uint64_t)high << 32)); 455 } 456 457 static __inline void 458 load_xcr(u_int reg, uint64_t val) 459 { 460 u_int low, high; 461 462 low = val; 463 high = val >> 32; 464 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 465 } 466 467 /* 468 * Global TLB flush (except for thise for pages marked PG_G) 469 */ 470 static __inline void 471 invltlb(void) 472 { 473 474 load_cr3(rcr3()); 475 } 476 477 /* 478 * TLB flush for an individual page (even if it has PG_G). 479 * Only works on 486+ CPUs (i386 does not have PG_G). 480 */ 481 static __inline void 482 invlpg(u_int addr) 483 { 484 485 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 486 } 487 488 static __inline u_short 489 rfs(void) 490 { 491 u_short sel; 492 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 493 return (sel); 494 } 495 496 static __inline uint64_t 497 rgdt(void) 498 { 499 uint64_t gdtr; 500 __asm __volatile("sgdt %0" : "=m" (gdtr)); 501 return (gdtr); 502 } 503 504 static __inline u_short 505 rgs(void) 506 { 507 u_short sel; 508 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 509 return (sel); 510 } 511 512 static __inline uint64_t 513 ridt(void) 514 { 515 uint64_t idtr; 516 __asm __volatile("sidt %0" : "=m" (idtr)); 517 return (idtr); 518 } 519 520 static __inline u_short 521 rldt(void) 522 { 523 u_short ldtr; 524 __asm __volatile("sldt %0" : "=g" (ldtr)); 525 return (ldtr); 526 } 527 528 static __inline u_short 529 rss(void) 530 { 531 u_short sel; 532 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 533 return (sel); 534 } 535 536 static __inline u_short 537 rtr(void) 538 { 539 u_short tr; 540 __asm __volatile("str %0" : "=g" (tr)); 541 return (tr); 542 } 543 544 static __inline void 545 load_fs(u_short sel) 546 { 547 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 548 } 549 550 static __inline void 551 load_gs(u_short sel) 552 { 553 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 554 } 555 556 static __inline void 557 lidt(struct region_descriptor *addr) 558 { 559 __asm __volatile("lidt (%0)" : : "r" (addr)); 560 } 561 562 static __inline void 563 lldt(u_short sel) 564 { 565 __asm __volatile("lldt %0" : : "r" (sel)); 566 } 567 568 static __inline void 569 ltr(u_short sel) 570 { 571 __asm __volatile("ltr %0" : : "r" (sel)); 572 } 573 574 static __inline u_int 575 rdr0(void) 576 { 577 u_int data; 578 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 579 return (data); 580 } 581 582 static __inline void 583 load_dr0(u_int dr0) 584 { 585 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 586 } 587 588 static __inline u_int 589 rdr1(void) 590 { 591 u_int data; 592 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 593 return (data); 594 } 595 596 static __inline void 597 load_dr1(u_int dr1) 598 { 599 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 600 } 601 602 static __inline u_int 603 rdr2(void) 604 { 605 u_int data; 606 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 607 return (data); 608 } 609 610 static __inline void 611 load_dr2(u_int dr2) 612 { 613 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 614 } 615 616 static __inline u_int 617 rdr3(void) 618 { 619 u_int data; 620 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 621 return (data); 622 } 623 624 static __inline void 625 load_dr3(u_int dr3) 626 { 627 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 628 } 629 630 static __inline u_int 631 rdr4(void) 632 { 633 u_int data; 634 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 635 return (data); 636 } 637 638 static __inline void 639 load_dr4(u_int dr4) 640 { 641 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 642 } 643 644 static __inline u_int 645 rdr5(void) 646 { 647 u_int data; 648 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 649 return (data); 650 } 651 652 static __inline void 653 load_dr5(u_int dr5) 654 { 655 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 656 } 657 658 static __inline u_int 659 rdr6(void) 660 { 661 u_int data; 662 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 663 return (data); 664 } 665 666 static __inline void 667 load_dr6(u_int dr6) 668 { 669 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 670 } 671 672 static __inline u_int 673 rdr7(void) 674 { 675 u_int data; 676 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 677 return (data); 678 } 679 680 static __inline void 681 load_dr7(u_int dr7) 682 { 683 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 684 } 685 686 static __inline u_char 687 read_cyrix_reg(u_char reg) 688 { 689 outb(0x22, reg); 690 return inb(0x23); 691 } 692 693 static __inline void 694 write_cyrix_reg(u_char reg, u_char data) 695 { 696 outb(0x22, reg); 697 outb(0x23, data); 698 } 699 700 static __inline register_t 701 intr_disable(void) 702 { 703 register_t eflags; 704 705 eflags = read_eflags(); 706 disable_intr(); 707 return (eflags); 708 } 709 710 static __inline void 711 intr_restore(register_t eflags) 712 { 713 write_eflags(eflags); 714 } 715 716 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 717 718 int breakpoint(void); 719 u_int bsfl(u_int mask); 720 u_int bsrl(u_int mask); 721 void clflush(u_long addr); 722 void clts(void); 723 void cpuid_count(u_int ax, u_int cx, u_int *p); 724 void disable_intr(void); 725 void do_cpuid(u_int ax, u_int *p); 726 void enable_intr(void); 727 void halt(void); 728 void ia32_pause(void); 729 u_char inb(u_int port); 730 u_int inl(u_int port); 731 void insb(u_int port, void *addr, size_t count); 732 void insl(u_int port, void *addr, size_t count); 733 void insw(u_int port, void *addr, size_t count); 734 register_t intr_disable(void); 735 void intr_restore(register_t ef); 736 void invd(void); 737 void invlpg(u_int addr); 738 void invltlb(void); 739 u_short inw(u_int port); 740 void lidt(struct region_descriptor *addr); 741 void lldt(u_short sel); 742 void load_cr0(u_int cr0); 743 void load_cr3(u_int cr3); 744 void load_cr4(u_int cr4); 745 void load_dr0(u_int dr0); 746 void load_dr1(u_int dr1); 747 void load_dr2(u_int dr2); 748 void load_dr3(u_int dr3); 749 void load_dr4(u_int dr4); 750 void load_dr5(u_int dr5); 751 void load_dr6(u_int dr6); 752 void load_dr7(u_int dr7); 753 void load_fs(u_short sel); 754 void load_gs(u_short sel); 755 void ltr(u_short sel); 756 void outb(u_int port, u_char data); 757 void outl(u_int port, u_int data); 758 void outsb(u_int port, const void *addr, size_t count); 759 void outsl(u_int port, const void *addr, size_t count); 760 void outsw(u_int port, const void *addr, size_t count); 761 void outw(u_int port, u_short data); 762 u_int rcr0(void); 763 u_int rcr2(void); 764 u_int rcr3(void); 765 u_int rcr4(void); 766 uint64_t rdmsr(u_int msr); 767 uint64_t rdpmc(u_int pmc); 768 u_int rdr0(void); 769 u_int rdr1(void); 770 u_int rdr2(void); 771 u_int rdr3(void); 772 u_int rdr4(void); 773 u_int rdr5(void); 774 u_int rdr6(void); 775 u_int rdr7(void); 776 uint64_t rdtsc(void); 777 u_char read_cyrix_reg(u_char reg); 778 u_int read_eflags(void); 779 u_int rfs(void); 780 uint64_t rgdt(void); 781 u_int rgs(void); 782 uint64_t ridt(void); 783 u_short rldt(void); 784 u_short rtr(void); 785 void wbinvd(void); 786 void write_cyrix_reg(u_char reg, u_char data); 787 void write_eflags(u_int ef); 788 void wrmsr(u_int msr, uint64_t newval); 789 790 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 791 792 void reset_dbregs(void); 793 794 #ifdef _KERNEL 795 int rdmsr_safe(u_int msr, uint64_t *val); 796 int wrmsr_safe(u_int msr, uint64_t newval); 797 #endif 798 799 #endif /* !_MACHINE_CPUFUNC_H_ */ 800