1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 /* 35 * Functions to provide access to special i386 instructions. 36 * This in included in sys/systm.h, and that file should be 37 * used in preference to this. 38 */ 39 40 #ifndef _MACHINE_CPUFUNC_H_ 41 #define _MACHINE_CPUFUNC_H_ 42 43 #ifndef _SYS_CDEFS_H_ 44 #error this file needs sys/cdefs.h as a prerequisite 45 #endif 46 47 struct region_descriptor; 48 49 #define readb(va) (*(volatile uint8_t *) (va)) 50 #define readw(va) (*(volatile uint16_t *) (va)) 51 #define readl(va) (*(volatile uint32_t *) (va)) 52 53 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 54 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 55 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 56 57 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 58 59 static __inline void 60 breakpoint(void) 61 { 62 __asm __volatile("int $3"); 63 } 64 65 static __inline __pure2 u_int 66 bsfl(u_int mask) 67 { 68 u_int result; 69 70 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 71 return (result); 72 } 73 74 static __inline __pure2 u_int 75 bsrl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline void 84 clflush(u_long addr) 85 { 86 87 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 88 } 89 90 static __inline void 91 clflushopt(u_long addr) 92 { 93 94 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 95 } 96 97 static __inline void 98 clts(void) 99 { 100 101 __asm __volatile("clts"); 102 } 103 104 static __inline void 105 disable_intr(void) 106 { 107 108 __asm __volatile("cli" : : : "memory"); 109 } 110 111 static __inline void 112 do_cpuid(u_int ax, u_int *p) 113 { 114 __asm __volatile("cpuid" 115 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 116 : "0" (ax)); 117 } 118 119 static __inline void 120 cpuid_count(u_int ax, u_int cx, u_int *p) 121 { 122 __asm __volatile("cpuid" 123 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 124 : "0" (ax), "c" (cx)); 125 } 126 127 static __inline void 128 enable_intr(void) 129 { 130 131 __asm __volatile("sti"); 132 } 133 134 static __inline void 135 cpu_monitor(const void *addr, u_long extensions, u_int hints) 136 { 137 138 __asm __volatile("monitor" 139 : : "a" (addr), "c" (extensions), "d" (hints)); 140 } 141 142 static __inline void 143 cpu_mwait(u_long extensions, u_int hints) 144 { 145 146 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 147 } 148 149 static __inline void 150 lfence(void) 151 { 152 153 __asm __volatile("lfence" : : : "memory"); 154 } 155 156 static __inline void 157 mfence(void) 158 { 159 160 __asm __volatile("mfence" : : : "memory"); 161 } 162 163 static __inline void 164 sfence(void) 165 { 166 167 __asm __volatile("sfence" : : : "memory"); 168 } 169 170 #ifdef _KERNEL 171 172 #define HAVE_INLINE_FFS 173 174 static __inline __pure2 int 175 ffs(int mask) 176 { 177 /* 178 * Note that gcc-2's builtin ffs would be used if we didn't declare 179 * this inline or turn off the builtin. The builtin is faster but 180 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 181 * versions. 182 */ 183 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 184 } 185 186 #define HAVE_INLINE_FFSL 187 188 static __inline __pure2 int 189 ffsl(long mask) 190 { 191 return (ffs((int)mask)); 192 } 193 194 #define HAVE_INLINE_FLS 195 196 static __inline __pure2 int 197 fls(int mask) 198 { 199 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 200 } 201 202 #define HAVE_INLINE_FLSL 203 204 static __inline __pure2 int 205 flsl(long mask) 206 { 207 return (fls((int)mask)); 208 } 209 210 #endif /* _KERNEL */ 211 212 static __inline void 213 halt(void) 214 { 215 __asm __volatile("hlt"); 216 } 217 218 static __inline u_char 219 inb(u_int port) 220 { 221 u_char data; 222 223 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 224 return (data); 225 } 226 227 static __inline u_int 228 inl(u_int port) 229 { 230 u_int data; 231 232 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 233 return (data); 234 } 235 236 static __inline void 237 insb(u_int port, void *addr, size_t count) 238 { 239 __asm __volatile("cld; rep; insb" 240 : "+D" (addr), "+c" (count) 241 : "d" (port) 242 : "memory"); 243 } 244 245 static __inline void 246 insw(u_int port, void *addr, size_t count) 247 { 248 __asm __volatile("cld; rep; insw" 249 : "+D" (addr), "+c" (count) 250 : "d" (port) 251 : "memory"); 252 } 253 254 static __inline void 255 insl(u_int port, void *addr, size_t count) 256 { 257 __asm __volatile("cld; rep; insl" 258 : "+D" (addr), "+c" (count) 259 : "d" (port) 260 : "memory"); 261 } 262 263 static __inline void 264 invd(void) 265 { 266 __asm __volatile("invd"); 267 } 268 269 static __inline u_short 270 inw(u_int port) 271 { 272 u_short data; 273 274 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 275 return (data); 276 } 277 278 static __inline void 279 outb(u_int port, u_char data) 280 { 281 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 282 } 283 284 static __inline void 285 outl(u_int port, u_int data) 286 { 287 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 288 } 289 290 static __inline void 291 outsb(u_int port, const void *addr, size_t count) 292 { 293 __asm __volatile("cld; rep; outsb" 294 : "+S" (addr), "+c" (count) 295 : "d" (port)); 296 } 297 298 static __inline void 299 outsw(u_int port, const void *addr, size_t count) 300 { 301 __asm __volatile("cld; rep; outsw" 302 : "+S" (addr), "+c" (count) 303 : "d" (port)); 304 } 305 306 static __inline void 307 outsl(u_int port, const void *addr, size_t count) 308 { 309 __asm __volatile("cld; rep; outsl" 310 : "+S" (addr), "+c" (count) 311 : "d" (port)); 312 } 313 314 static __inline void 315 outw(u_int port, u_short data) 316 { 317 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 318 } 319 320 static __inline void 321 ia32_pause(void) 322 { 323 __asm __volatile("pause"); 324 } 325 326 static __inline u_int 327 read_eflags(void) 328 { 329 u_int ef; 330 331 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 332 return (ef); 333 } 334 335 static __inline uint64_t 336 rdmsr(u_int msr) 337 { 338 uint64_t rv; 339 340 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 341 return (rv); 342 } 343 344 static __inline uint32_t 345 rdmsr32(u_int msr) 346 { 347 uint32_t low; 348 349 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 350 return (low); 351 } 352 353 static __inline uint64_t 354 rdpmc(u_int pmc) 355 { 356 uint64_t rv; 357 358 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 359 return (rv); 360 } 361 362 static __inline uint64_t 363 rdtsc(void) 364 { 365 uint64_t rv; 366 367 __asm __volatile("rdtsc" : "=A" (rv)); 368 return (rv); 369 } 370 371 static __inline uint64_t 372 rdtscp(void) 373 { 374 uint64_t rv; 375 376 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx"); 377 return (rv); 378 } 379 380 static __inline uint32_t 381 rdtsc32(void) 382 { 383 uint32_t rv; 384 385 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 386 return (rv); 387 } 388 389 static __inline void 390 wbinvd(void) 391 { 392 __asm __volatile("wbinvd"); 393 } 394 395 static __inline void 396 write_eflags(u_int ef) 397 { 398 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 399 } 400 401 static __inline void 402 wrmsr(u_int msr, uint64_t newval) 403 { 404 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 405 } 406 407 static __inline void 408 load_cr0(u_int data) 409 { 410 411 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 412 } 413 414 static __inline u_int 415 rcr0(void) 416 { 417 u_int data; 418 419 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 420 return (data); 421 } 422 423 static __inline u_int 424 rcr2(void) 425 { 426 u_int data; 427 428 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 429 return (data); 430 } 431 432 static __inline void 433 load_cr3(u_int data) 434 { 435 436 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 437 } 438 439 static __inline u_int 440 rcr3(void) 441 { 442 u_int data; 443 444 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 445 return (data); 446 } 447 448 static __inline void 449 load_cr4(u_int data) 450 { 451 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 452 } 453 454 static __inline u_int 455 rcr4(void) 456 { 457 u_int data; 458 459 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 460 return (data); 461 } 462 463 static __inline uint64_t 464 rxcr(u_int reg) 465 { 466 u_int low, high; 467 468 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 469 return (low | ((uint64_t)high << 32)); 470 } 471 472 static __inline void 473 load_xcr(u_int reg, uint64_t val) 474 { 475 u_int low, high; 476 477 low = val; 478 high = val >> 32; 479 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 480 } 481 482 /* 483 * Global TLB flush (except for thise for pages marked PG_G) 484 */ 485 static __inline void 486 invltlb(void) 487 { 488 489 load_cr3(rcr3()); 490 } 491 492 /* 493 * TLB flush for an individual page (even if it has PG_G). 494 * Only works on 486+ CPUs (i386 does not have PG_G). 495 */ 496 static __inline void 497 invlpg(u_int addr) 498 { 499 500 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 501 } 502 503 static __inline u_short 504 rfs(void) 505 { 506 u_short sel; 507 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 508 return (sel); 509 } 510 511 static __inline uint64_t 512 rgdt(void) 513 { 514 uint64_t gdtr; 515 __asm __volatile("sgdt %0" : "=m" (gdtr)); 516 return (gdtr); 517 } 518 519 static __inline u_short 520 rgs(void) 521 { 522 u_short sel; 523 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 524 return (sel); 525 } 526 527 static __inline uint64_t 528 ridt(void) 529 { 530 uint64_t idtr; 531 __asm __volatile("sidt %0" : "=m" (idtr)); 532 return (idtr); 533 } 534 535 static __inline u_short 536 rldt(void) 537 { 538 u_short ldtr; 539 __asm __volatile("sldt %0" : "=g" (ldtr)); 540 return (ldtr); 541 } 542 543 static __inline u_short 544 rss(void) 545 { 546 u_short sel; 547 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 548 return (sel); 549 } 550 551 static __inline u_short 552 rtr(void) 553 { 554 u_short tr; 555 __asm __volatile("str %0" : "=g" (tr)); 556 return (tr); 557 } 558 559 static __inline void 560 load_fs(u_short sel) 561 { 562 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 563 } 564 565 static __inline void 566 load_gs(u_short sel) 567 { 568 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 569 } 570 571 static __inline void 572 lidt(struct region_descriptor *addr) 573 { 574 __asm __volatile("lidt (%0)" : : "r" (addr)); 575 } 576 577 static __inline void 578 lldt(u_short sel) 579 { 580 __asm __volatile("lldt %0" : : "r" (sel)); 581 } 582 583 static __inline void 584 ltr(u_short sel) 585 { 586 __asm __volatile("ltr %0" : : "r" (sel)); 587 } 588 589 static __inline u_int 590 rdr0(void) 591 { 592 u_int data; 593 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 594 return (data); 595 } 596 597 static __inline void 598 load_dr0(u_int dr0) 599 { 600 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 601 } 602 603 static __inline u_int 604 rdr1(void) 605 { 606 u_int data; 607 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 608 return (data); 609 } 610 611 static __inline void 612 load_dr1(u_int dr1) 613 { 614 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 615 } 616 617 static __inline u_int 618 rdr2(void) 619 { 620 u_int data; 621 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 622 return (data); 623 } 624 625 static __inline void 626 load_dr2(u_int dr2) 627 { 628 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 629 } 630 631 static __inline u_int 632 rdr3(void) 633 { 634 u_int data; 635 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 636 return (data); 637 } 638 639 static __inline void 640 load_dr3(u_int dr3) 641 { 642 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 643 } 644 645 static __inline u_int 646 rdr6(void) 647 { 648 u_int data; 649 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 650 return (data); 651 } 652 653 static __inline void 654 load_dr6(u_int dr6) 655 { 656 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 657 } 658 659 static __inline u_int 660 rdr7(void) 661 { 662 u_int data; 663 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 664 return (data); 665 } 666 667 static __inline void 668 load_dr7(u_int dr7) 669 { 670 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 671 } 672 673 static __inline u_char 674 read_cyrix_reg(u_char reg) 675 { 676 outb(0x22, reg); 677 return inb(0x23); 678 } 679 680 static __inline void 681 write_cyrix_reg(u_char reg, u_char data) 682 { 683 outb(0x22, reg); 684 outb(0x23, data); 685 } 686 687 static __inline register_t 688 intr_disable(void) 689 { 690 register_t eflags; 691 692 eflags = read_eflags(); 693 disable_intr(); 694 return (eflags); 695 } 696 697 static __inline void 698 intr_restore(register_t eflags) 699 { 700 write_eflags(eflags); 701 } 702 703 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 704 705 int breakpoint(void); 706 u_int bsfl(u_int mask); 707 u_int bsrl(u_int mask); 708 void clflush(u_long addr); 709 void clts(void); 710 void cpuid_count(u_int ax, u_int cx, u_int *p); 711 void disable_intr(void); 712 void do_cpuid(u_int ax, u_int *p); 713 void enable_intr(void); 714 void halt(void); 715 void ia32_pause(void); 716 u_char inb(u_int port); 717 u_int inl(u_int port); 718 void insb(u_int port, void *addr, size_t count); 719 void insl(u_int port, void *addr, size_t count); 720 void insw(u_int port, void *addr, size_t count); 721 register_t intr_disable(void); 722 void intr_restore(register_t ef); 723 void invd(void); 724 void invlpg(u_int addr); 725 void invltlb(void); 726 u_short inw(u_int port); 727 void lidt(struct region_descriptor *addr); 728 void lldt(u_short sel); 729 void load_cr0(u_int cr0); 730 void load_cr3(u_int cr3); 731 void load_cr4(u_int cr4); 732 void load_dr0(u_int dr0); 733 void load_dr1(u_int dr1); 734 void load_dr2(u_int dr2); 735 void load_dr3(u_int dr3); 736 void load_dr6(u_int dr6); 737 void load_dr7(u_int dr7); 738 void load_fs(u_short sel); 739 void load_gs(u_short sel); 740 void ltr(u_short sel); 741 void outb(u_int port, u_char data); 742 void outl(u_int port, u_int data); 743 void outsb(u_int port, const void *addr, size_t count); 744 void outsl(u_int port, const void *addr, size_t count); 745 void outsw(u_int port, const void *addr, size_t count); 746 void outw(u_int port, u_short data); 747 u_int rcr0(void); 748 u_int rcr2(void); 749 u_int rcr3(void); 750 u_int rcr4(void); 751 uint64_t rdmsr(u_int msr); 752 uint64_t rdpmc(u_int pmc); 753 u_int rdr0(void); 754 u_int rdr1(void); 755 u_int rdr2(void); 756 u_int rdr3(void); 757 u_int rdr6(void); 758 u_int rdr7(void); 759 uint64_t rdtsc(void); 760 u_char read_cyrix_reg(u_char reg); 761 u_int read_eflags(void); 762 u_int rfs(void); 763 uint64_t rgdt(void); 764 u_int rgs(void); 765 uint64_t ridt(void); 766 u_short rldt(void); 767 u_short rtr(void); 768 void wbinvd(void); 769 void write_cyrix_reg(u_char reg, u_char data); 770 void write_eflags(u_int ef); 771 void wrmsr(u_int msr, uint64_t newval); 772 773 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 774 775 void reset_dbregs(void); 776 777 #ifdef _KERNEL 778 int rdmsr_safe(u_int msr, uint64_t *val); 779 int wrmsr_safe(u_int msr, uint64_t newval); 780 #endif 781 782 #endif /* !_MACHINE_CPUFUNC_H_ */ 783