1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 /* 35 * Functions to provide access to special i386 instructions. 36 * This in included in sys/systm.h, and that file should be 37 * used in preference to this. 38 */ 39 40 #ifndef _MACHINE_CPUFUNC_H_ 41 #define _MACHINE_CPUFUNC_H_ 42 43 #ifndef _SYS_CDEFS_H_ 44 #error this file needs sys/cdefs.h as a prerequisite 45 #endif 46 47 struct region_descriptor; 48 49 #define readb(va) (*(volatile uint8_t *) (va)) 50 #define readw(va) (*(volatile uint16_t *) (va)) 51 #define readl(va) (*(volatile uint32_t *) (va)) 52 53 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 54 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 55 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 56 57 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 58 59 static __inline void 60 breakpoint(void) 61 { 62 __asm __volatile("int $3"); 63 } 64 65 static __inline __pure2 u_int 66 bsfl(u_int mask) 67 { 68 u_int result; 69 70 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 71 return (result); 72 } 73 74 static __inline __pure2 u_int 75 bsrl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline void 84 clflush(u_long addr) 85 { 86 87 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 88 } 89 90 static __inline void 91 clflushopt(u_long addr) 92 { 93 94 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 95 } 96 97 static __inline void 98 clts(void) 99 { 100 101 __asm __volatile("clts"); 102 } 103 104 static __inline void 105 disable_intr(void) 106 { 107 108 __asm __volatile("cli" : : : "memory"); 109 } 110 111 #ifdef _KERNEL 112 static __inline void 113 do_cpuid(u_int ax, u_int *p) 114 { 115 __asm __volatile("cpuid" 116 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 117 : "0" (ax)); 118 } 119 120 static __inline void 121 cpuid_count(u_int ax, u_int cx, u_int *p) 122 { 123 __asm __volatile("cpuid" 124 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 125 : "0" (ax), "c" (cx)); 126 } 127 #else 128 static __inline void 129 do_cpuid(u_int ax, u_int *p) 130 { 131 __asm __volatile( 132 "pushl\t%%ebx\n\t" 133 "cpuid\n\t" 134 "movl\t%%ebx,%1\n\t" 135 "popl\t%%ebx" 136 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 137 : "0" (ax)); 138 } 139 140 static __inline void 141 cpuid_count(u_int ax, u_int cx, u_int *p) 142 { 143 __asm __volatile( 144 "pushl\t%%ebx\n\t" 145 "cpuid\n\t" 146 "movl\t%%ebx,%1\n\t" 147 "popl\t%%ebx" 148 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 149 : "0" (ax), "c" (cx)); 150 } 151 #endif 152 153 static __inline void 154 enable_intr(void) 155 { 156 157 __asm __volatile("sti"); 158 } 159 160 static __inline void 161 cpu_monitor(const void *addr, u_long extensions, u_int hints) 162 { 163 164 __asm __volatile("monitor" 165 : : "a" (addr), "c" (extensions), "d" (hints)); 166 } 167 168 static __inline void 169 cpu_mwait(u_long extensions, u_int hints) 170 { 171 172 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 173 } 174 175 static __inline void 176 lfence(void) 177 { 178 179 __asm __volatile("lfence" : : : "memory"); 180 } 181 182 static __inline void 183 mfence(void) 184 { 185 186 __asm __volatile("mfence" : : : "memory"); 187 } 188 189 static __inline void 190 sfence(void) 191 { 192 193 __asm __volatile("sfence" : : : "memory"); 194 } 195 196 #ifdef _KERNEL 197 198 #define HAVE_INLINE_FFS 199 200 static __inline __pure2 int 201 ffs(int mask) 202 { 203 /* 204 * Note that gcc-2's builtin ffs would be used if we didn't declare 205 * this inline or turn off the builtin. The builtin is faster but 206 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 207 * versions. 208 */ 209 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 210 } 211 212 #define HAVE_INLINE_FFSL 213 214 static __inline __pure2 int 215 ffsl(long mask) 216 { 217 return (ffs((int)mask)); 218 } 219 220 #define HAVE_INLINE_FLS 221 222 static __inline __pure2 int 223 fls(int mask) 224 { 225 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 226 } 227 228 #define HAVE_INLINE_FLSL 229 230 static __inline __pure2 int 231 flsl(long mask) 232 { 233 return (fls((int)mask)); 234 } 235 236 #endif /* _KERNEL */ 237 238 static __inline void 239 halt(void) 240 { 241 __asm __volatile("hlt"); 242 } 243 244 static __inline u_char 245 inb(u_int port) 246 { 247 u_char data; 248 249 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 250 return (data); 251 } 252 253 static __inline u_int 254 inl(u_int port) 255 { 256 u_int data; 257 258 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 259 return (data); 260 } 261 262 static __inline void 263 insb(u_int port, void *addr, size_t count) 264 { 265 __asm __volatile("cld; rep; insb" 266 : "+D" (addr), "+c" (count) 267 : "d" (port) 268 : "memory"); 269 } 270 271 static __inline void 272 insw(u_int port, void *addr, size_t count) 273 { 274 __asm __volatile("cld; rep; insw" 275 : "+D" (addr), "+c" (count) 276 : "d" (port) 277 : "memory"); 278 } 279 280 static __inline void 281 insl(u_int port, void *addr, size_t count) 282 { 283 __asm __volatile("cld; rep; insl" 284 : "+D" (addr), "+c" (count) 285 : "d" (port) 286 : "memory"); 287 } 288 289 static __inline void 290 invd(void) 291 { 292 __asm __volatile("invd"); 293 } 294 295 static __inline u_short 296 inw(u_int port) 297 { 298 u_short data; 299 300 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 301 return (data); 302 } 303 304 static __inline void 305 outb(u_int port, u_char data) 306 { 307 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 308 } 309 310 static __inline void 311 outl(u_int port, u_int data) 312 { 313 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 314 } 315 316 static __inline void 317 outsb(u_int port, const void *addr, size_t count) 318 { 319 __asm __volatile("cld; rep; outsb" 320 : "+S" (addr), "+c" (count) 321 : "d" (port)); 322 } 323 324 static __inline void 325 outsw(u_int port, const void *addr, size_t count) 326 { 327 __asm __volatile("cld; rep; outsw" 328 : "+S" (addr), "+c" (count) 329 : "d" (port)); 330 } 331 332 static __inline void 333 outsl(u_int port, const void *addr, size_t count) 334 { 335 __asm __volatile("cld; rep; outsl" 336 : "+S" (addr), "+c" (count) 337 : "d" (port)); 338 } 339 340 static __inline void 341 outw(u_int port, u_short data) 342 { 343 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 344 } 345 346 static __inline void 347 ia32_pause(void) 348 { 349 __asm __volatile("pause"); 350 } 351 352 static __inline u_int 353 read_eflags(void) 354 { 355 u_int ef; 356 357 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 358 return (ef); 359 } 360 361 static __inline uint64_t 362 rdmsr(u_int msr) 363 { 364 uint64_t rv; 365 366 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 367 return (rv); 368 } 369 370 static __inline uint32_t 371 rdmsr32(u_int msr) 372 { 373 uint32_t low; 374 375 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 376 return (low); 377 } 378 379 static __inline uint64_t 380 rdpmc(u_int pmc) 381 { 382 uint64_t rv; 383 384 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 385 return (rv); 386 } 387 388 static __inline uint64_t 389 rdtsc(void) 390 { 391 uint64_t rv; 392 393 __asm __volatile("rdtsc" : "=A" (rv)); 394 return (rv); 395 } 396 397 static __inline uint64_t 398 rdtsc_ordered_lfence(void) 399 { 400 lfence(); 401 return (rdtsc()); 402 } 403 404 static __inline uint64_t 405 rdtsc_ordered_mfence(void) 406 { 407 mfence(); 408 return (rdtsc()); 409 } 410 411 static __inline uint64_t 412 rdtscp(void) 413 { 414 uint64_t rv; 415 416 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx"); 417 return (rv); 418 } 419 420 static __inline uint64_t 421 rdtscp_aux(uint32_t *aux) 422 { 423 uint64_t rv; 424 425 __asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux)); 426 return (rv); 427 } 428 429 static __inline uint32_t 430 rdtsc32(void) 431 { 432 uint32_t rv; 433 434 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 435 return (rv); 436 } 437 438 static __inline uint32_t 439 rdtscp32(void) 440 { 441 uint32_t rv; 442 443 __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx"); 444 return (rv); 445 } 446 447 static __inline void 448 wbinvd(void) 449 { 450 __asm __volatile("wbinvd"); 451 } 452 453 static __inline void 454 write_eflags(u_int ef) 455 { 456 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 457 } 458 459 static __inline void 460 wrmsr(u_int msr, uint64_t newval) 461 { 462 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 463 } 464 465 static __inline void 466 load_cr0(u_int data) 467 { 468 469 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 470 } 471 472 static __inline u_int 473 rcr0(void) 474 { 475 u_int data; 476 477 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 478 return (data); 479 } 480 481 static __inline u_int 482 rcr2(void) 483 { 484 u_int data; 485 486 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 487 return (data); 488 } 489 490 static __inline void 491 load_cr3(u_int data) 492 { 493 494 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 495 } 496 497 static __inline u_int 498 rcr3(void) 499 { 500 u_int data; 501 502 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 503 return (data); 504 } 505 506 static __inline void 507 load_cr4(u_int data) 508 { 509 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 510 } 511 512 static __inline u_int 513 rcr4(void) 514 { 515 u_int data; 516 517 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 518 return (data); 519 } 520 521 static __inline uint64_t 522 rxcr(u_int reg) 523 { 524 u_int low, high; 525 526 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 527 return (low | ((uint64_t)high << 32)); 528 } 529 530 static __inline void 531 load_xcr(u_int reg, uint64_t val) 532 { 533 u_int low, high; 534 535 low = val; 536 high = val >> 32; 537 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 538 } 539 540 /* 541 * Global TLB flush (except for thise for pages marked PG_G) 542 */ 543 static __inline void 544 invltlb(void) 545 { 546 547 load_cr3(rcr3()); 548 } 549 550 /* 551 * TLB flush for an individual page (even if it has PG_G). 552 * Only works on 486+ CPUs (i386 does not have PG_G). 553 */ 554 static __inline void 555 invlpg(u_int addr) 556 { 557 558 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 559 } 560 561 static __inline u_short 562 rfs(void) 563 { 564 u_short sel; 565 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 566 return (sel); 567 } 568 569 static __inline uint64_t 570 rgdt(void) 571 { 572 uint64_t gdtr; 573 __asm __volatile("sgdt %0" : "=m" (gdtr)); 574 return (gdtr); 575 } 576 577 static __inline u_short 578 rgs(void) 579 { 580 u_short sel; 581 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 582 return (sel); 583 } 584 585 static __inline uint64_t 586 ridt(void) 587 { 588 uint64_t idtr; 589 __asm __volatile("sidt %0" : "=m" (idtr)); 590 return (idtr); 591 } 592 593 static __inline u_short 594 rldt(void) 595 { 596 u_short ldtr; 597 __asm __volatile("sldt %0" : "=g" (ldtr)); 598 return (ldtr); 599 } 600 601 static __inline u_short 602 rss(void) 603 { 604 u_short sel; 605 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 606 return (sel); 607 } 608 609 static __inline u_short 610 rtr(void) 611 { 612 u_short tr; 613 __asm __volatile("str %0" : "=g" (tr)); 614 return (tr); 615 } 616 617 static __inline void 618 load_fs(u_short sel) 619 { 620 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 621 } 622 623 static __inline void 624 load_gs(u_short sel) 625 { 626 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 627 } 628 629 static __inline void 630 lidt(struct region_descriptor *addr) 631 { 632 __asm __volatile("lidt (%0)" : : "r" (addr)); 633 } 634 635 static __inline void 636 lldt(u_short sel) 637 { 638 __asm __volatile("lldt %0" : : "r" (sel)); 639 } 640 641 static __inline void 642 ltr(u_short sel) 643 { 644 __asm __volatile("ltr %0" : : "r" (sel)); 645 } 646 647 static __inline u_int 648 rdr0(void) 649 { 650 u_int data; 651 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 652 return (data); 653 } 654 655 static __inline void 656 load_dr0(u_int dr0) 657 { 658 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 659 } 660 661 static __inline u_int 662 rdr1(void) 663 { 664 u_int data; 665 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 666 return (data); 667 } 668 669 static __inline void 670 load_dr1(u_int dr1) 671 { 672 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 673 } 674 675 static __inline u_int 676 rdr2(void) 677 { 678 u_int data; 679 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 680 return (data); 681 } 682 683 static __inline void 684 load_dr2(u_int dr2) 685 { 686 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 687 } 688 689 static __inline u_int 690 rdr3(void) 691 { 692 u_int data; 693 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 694 return (data); 695 } 696 697 static __inline void 698 load_dr3(u_int dr3) 699 { 700 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 701 } 702 703 static __inline u_int 704 rdr6(void) 705 { 706 u_int data; 707 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 708 return (data); 709 } 710 711 static __inline void 712 load_dr6(u_int dr6) 713 { 714 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 715 } 716 717 static __inline u_int 718 rdr7(void) 719 { 720 u_int data; 721 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 722 return (data); 723 } 724 725 static __inline void 726 load_dr7(u_int dr7) 727 { 728 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 729 } 730 731 static __inline u_char 732 read_cyrix_reg(u_char reg) 733 { 734 outb(0x22, reg); 735 return inb(0x23); 736 } 737 738 static __inline void 739 write_cyrix_reg(u_char reg, u_char data) 740 { 741 outb(0x22, reg); 742 outb(0x23, data); 743 } 744 745 static __inline register_t 746 intr_disable(void) 747 { 748 register_t eflags; 749 750 eflags = read_eflags(); 751 disable_intr(); 752 return (eflags); 753 } 754 755 static __inline void 756 intr_restore(register_t eflags) 757 { 758 write_eflags(eflags); 759 } 760 761 static __inline uint32_t 762 rdpkru(void) 763 { 764 uint32_t res; 765 766 __asm __volatile("rdpkru" : "=a" (res) : "c" (0) : "edx"); 767 return (res); 768 } 769 770 static __inline void 771 wrpkru(uint32_t mask) 772 { 773 774 __asm __volatile("wrpkru" : : "a" (mask), "c" (0), "d" (0)); 775 } 776 777 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 778 779 int breakpoint(void); 780 u_int bsfl(u_int mask); 781 u_int bsrl(u_int mask); 782 void clflush(u_long addr); 783 void clts(void); 784 void cpuid_count(u_int ax, u_int cx, u_int *p); 785 void disable_intr(void); 786 void do_cpuid(u_int ax, u_int *p); 787 void enable_intr(void); 788 void halt(void); 789 void ia32_pause(void); 790 u_char inb(u_int port); 791 u_int inl(u_int port); 792 void insb(u_int port, void *addr, size_t count); 793 void insl(u_int port, void *addr, size_t count); 794 void insw(u_int port, void *addr, size_t count); 795 register_t intr_disable(void); 796 void intr_restore(register_t ef); 797 void invd(void); 798 void invlpg(u_int addr); 799 void invltlb(void); 800 u_short inw(u_int port); 801 void lidt(struct region_descriptor *addr); 802 void lldt(u_short sel); 803 void load_cr0(u_int cr0); 804 void load_cr3(u_int cr3); 805 void load_cr4(u_int cr4); 806 void load_dr0(u_int dr0); 807 void load_dr1(u_int dr1); 808 void load_dr2(u_int dr2); 809 void load_dr3(u_int dr3); 810 void load_dr6(u_int dr6); 811 void load_dr7(u_int dr7); 812 void load_fs(u_short sel); 813 void load_gs(u_short sel); 814 void ltr(u_short sel); 815 void outb(u_int port, u_char data); 816 void outl(u_int port, u_int data); 817 void outsb(u_int port, const void *addr, size_t count); 818 void outsl(u_int port, const void *addr, size_t count); 819 void outsw(u_int port, const void *addr, size_t count); 820 void outw(u_int port, u_short data); 821 u_int rcr0(void); 822 u_int rcr2(void); 823 u_int rcr3(void); 824 u_int rcr4(void); 825 uint64_t rdmsr(u_int msr); 826 uint64_t rdpmc(u_int pmc); 827 u_int rdr0(void); 828 u_int rdr1(void); 829 u_int rdr2(void); 830 u_int rdr3(void); 831 u_int rdr6(void); 832 u_int rdr7(void); 833 uint64_t rdtsc(void); 834 u_char read_cyrix_reg(u_char reg); 835 u_int read_eflags(void); 836 u_int rfs(void); 837 uint64_t rgdt(void); 838 u_int rgs(void); 839 uint64_t ridt(void); 840 u_short rldt(void); 841 u_short rtr(void); 842 void wbinvd(void); 843 void write_cyrix_reg(u_char reg, u_char data); 844 void write_eflags(u_int ef); 845 void wrmsr(u_int msr, uint64_t newval); 846 847 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 848 849 void reset_dbregs(void); 850 851 #ifdef _KERNEL 852 int rdmsr_safe(u_int msr, uint64_t *val); 853 int wrmsr_safe(u_int msr, uint64_t newval); 854 #endif 855 856 #endif /* !_MACHINE_CPUFUNC_H_ */ 857