1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern void write_eflags(u_int eflags); 53 extern u_int read_eflags(void); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile uint8_t *) (va)) 59 #define readw(va) (*(volatile uint16_t *) (va)) 60 #define readl(va) (*(volatile uint32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90 } 91 92 static __inline void 93 clflush(u_long addr) 94 { 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97 } 98 99 static __inline void 100 clts(void) 101 { 102 103 __asm __volatile("clts"); 104 } 105 106 static __inline void 107 disable_intr(void) 108 { 109 #ifdef XEN 110 xen_cli(); 111 #else 112 __asm __volatile("cli" : : : "memory"); 113 #endif 114 } 115 116 static __inline void 117 do_cpuid(u_int ax, u_int *p) 118 { 119 __asm __volatile("cpuid" 120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 121 : "0" (ax)); 122 } 123 124 static __inline void 125 cpuid_count(u_int ax, u_int cx, u_int *p) 126 { 127 __asm __volatile("cpuid" 128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 129 : "0" (ax), "c" (cx)); 130 } 131 132 static __inline void 133 enable_intr(void) 134 { 135 #ifdef XEN 136 xen_sti(); 137 #else 138 __asm __volatile("sti"); 139 #endif 140 } 141 142 static __inline void 143 cpu_monitor(const void *addr, u_long extensions, u_int hints) 144 { 145 146 __asm __volatile("monitor" 147 : : "a" (addr), "c" (extensions), "d" (hints)); 148 } 149 150 static __inline void 151 cpu_mwait(u_long extensions, u_int hints) 152 { 153 154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 155 } 156 157 static __inline void 158 lfence(void) 159 { 160 161 __asm __volatile("lfence" : : : "memory"); 162 } 163 164 static __inline void 165 mfence(void) 166 { 167 168 __asm __volatile("mfence" : : : "memory"); 169 } 170 171 #ifdef _KERNEL 172 173 #define HAVE_INLINE_FFS 174 175 static __inline int 176 ffs(int mask) 177 { 178 /* 179 * Note that gcc-2's builtin ffs would be used if we didn't declare 180 * this inline or turn off the builtin. The builtin is faster but 181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 182 * versions. 183 */ 184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 185 } 186 187 #define HAVE_INLINE_FLS 188 189 static __inline int 190 fls(int mask) 191 { 192 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 193 } 194 195 #endif /* _KERNEL */ 196 197 static __inline void 198 halt(void) 199 { 200 __asm __volatile("hlt"); 201 } 202 203 static __inline u_char 204 inb(u_int port) 205 { 206 u_char data; 207 208 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 209 return (data); 210 } 211 212 static __inline u_int 213 inl(u_int port) 214 { 215 u_int data; 216 217 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 218 return (data); 219 } 220 221 static __inline void 222 insb(u_int port, void *addr, size_t count) 223 { 224 __asm __volatile("cld; rep; insb" 225 : "+D" (addr), "+c" (count) 226 : "d" (port) 227 : "memory"); 228 } 229 230 static __inline void 231 insw(u_int port, void *addr, size_t count) 232 { 233 __asm __volatile("cld; rep; insw" 234 : "+D" (addr), "+c" (count) 235 : "d" (port) 236 : "memory"); 237 } 238 239 static __inline void 240 insl(u_int port, void *addr, size_t count) 241 { 242 __asm __volatile("cld; rep; insl" 243 : "+D" (addr), "+c" (count) 244 : "d" (port) 245 : "memory"); 246 } 247 248 static __inline void 249 invd(void) 250 { 251 __asm __volatile("invd"); 252 } 253 254 static __inline u_short 255 inw(u_int port) 256 { 257 u_short data; 258 259 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 260 return (data); 261 } 262 263 static __inline void 264 outb(u_int port, u_char data) 265 { 266 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 267 } 268 269 static __inline void 270 outl(u_int port, u_int data) 271 { 272 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 273 } 274 275 static __inline void 276 outsb(u_int port, const void *addr, size_t count) 277 { 278 __asm __volatile("cld; rep; outsb" 279 : "+S" (addr), "+c" (count) 280 : "d" (port)); 281 } 282 283 static __inline void 284 outsw(u_int port, const void *addr, size_t count) 285 { 286 __asm __volatile("cld; rep; outsw" 287 : "+S" (addr), "+c" (count) 288 : "d" (port)); 289 } 290 291 static __inline void 292 outsl(u_int port, const void *addr, size_t count) 293 { 294 __asm __volatile("cld; rep; outsl" 295 : "+S" (addr), "+c" (count) 296 : "d" (port)); 297 } 298 299 static __inline void 300 outw(u_int port, u_short data) 301 { 302 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 303 } 304 305 static __inline void 306 ia32_pause(void) 307 { 308 __asm __volatile("pause"); 309 } 310 311 static __inline u_int 312 #ifdef XEN 313 _read_eflags(void) 314 #else 315 read_eflags(void) 316 #endif 317 { 318 u_int ef; 319 320 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 321 return (ef); 322 } 323 324 static __inline uint64_t 325 rdmsr(u_int msr) 326 { 327 uint64_t rv; 328 329 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 330 return (rv); 331 } 332 333 static __inline uint64_t 334 rdpmc(u_int pmc) 335 { 336 uint64_t rv; 337 338 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 339 return (rv); 340 } 341 342 static __inline uint64_t 343 rdtsc(void) 344 { 345 uint64_t rv; 346 347 __asm __volatile("rdtsc" : "=A" (rv)); 348 return (rv); 349 } 350 351 static __inline uint32_t 352 rdtsc32(void) 353 { 354 uint32_t rv; 355 356 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 357 return (rv); 358 } 359 360 static __inline void 361 wbinvd(void) 362 { 363 __asm __volatile("wbinvd"); 364 } 365 366 static __inline void 367 #ifdef XEN 368 _write_eflags(u_int ef) 369 #else 370 write_eflags(u_int ef) 371 #endif 372 { 373 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 374 } 375 376 static __inline void 377 wrmsr(u_int msr, uint64_t newval) 378 { 379 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 380 } 381 382 static __inline void 383 load_cr0(u_int data) 384 { 385 386 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 387 } 388 389 static __inline u_int 390 rcr0(void) 391 { 392 u_int data; 393 394 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 395 return (data); 396 } 397 398 static __inline u_int 399 rcr2(void) 400 { 401 u_int data; 402 403 #ifdef XEN 404 return (xen_rcr2()); 405 #endif 406 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 407 return (data); 408 } 409 410 static __inline void 411 load_cr3(u_int data) 412 { 413 #ifdef XEN 414 xen_load_cr3(data); 415 #else 416 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 417 #endif 418 } 419 420 static __inline u_int 421 rcr3(void) 422 { 423 u_int data; 424 425 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 426 return (data); 427 } 428 429 static __inline void 430 load_cr4(u_int data) 431 { 432 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 433 } 434 435 static __inline u_int 436 rcr4(void) 437 { 438 u_int data; 439 440 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 441 return (data); 442 } 443 444 /* 445 * Global TLB flush (except for thise for pages marked PG_G) 446 */ 447 static __inline void 448 invltlb(void) 449 { 450 #ifdef XEN 451 xen_tlb_flush(); 452 #else 453 load_cr3(rcr3()); 454 #endif 455 } 456 457 /* 458 * TLB flush for an individual page (even if it has PG_G). 459 * Only works on 486+ CPUs (i386 does not have PG_G). 460 */ 461 static __inline void 462 invlpg(u_int addr) 463 { 464 465 #ifdef XEN 466 xen_invlpg(addr); 467 #else 468 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 469 #endif 470 } 471 472 static __inline u_short 473 rfs(void) 474 { 475 u_short sel; 476 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 477 return (sel); 478 } 479 480 static __inline uint64_t 481 rgdt(void) 482 { 483 uint64_t gdtr; 484 __asm __volatile("sgdt %0" : "=m" (gdtr)); 485 return (gdtr); 486 } 487 488 static __inline u_short 489 rgs(void) 490 { 491 u_short sel; 492 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 493 return (sel); 494 } 495 496 static __inline uint64_t 497 ridt(void) 498 { 499 uint64_t idtr; 500 __asm __volatile("sidt %0" : "=m" (idtr)); 501 return (idtr); 502 } 503 504 static __inline u_short 505 rldt(void) 506 { 507 u_short ldtr; 508 __asm __volatile("sldt %0" : "=g" (ldtr)); 509 return (ldtr); 510 } 511 512 static __inline u_short 513 rss(void) 514 { 515 u_short sel; 516 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 517 return (sel); 518 } 519 520 static __inline u_short 521 rtr(void) 522 { 523 u_short tr; 524 __asm __volatile("str %0" : "=g" (tr)); 525 return (tr); 526 } 527 528 static __inline void 529 load_fs(u_short sel) 530 { 531 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 532 } 533 534 static __inline void 535 load_gs(u_short sel) 536 { 537 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 538 } 539 540 static __inline void 541 lidt(struct region_descriptor *addr) 542 { 543 __asm __volatile("lidt (%0)" : : "r" (addr)); 544 } 545 546 static __inline void 547 lldt(u_short sel) 548 { 549 __asm __volatile("lldt %0" : : "r" (sel)); 550 } 551 552 static __inline void 553 ltr(u_short sel) 554 { 555 __asm __volatile("ltr %0" : : "r" (sel)); 556 } 557 558 static __inline u_int 559 rdr0(void) 560 { 561 u_int data; 562 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 563 return (data); 564 } 565 566 static __inline void 567 load_dr0(u_int dr0) 568 { 569 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 570 } 571 572 static __inline u_int 573 rdr1(void) 574 { 575 u_int data; 576 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 577 return (data); 578 } 579 580 static __inline void 581 load_dr1(u_int dr1) 582 { 583 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 584 } 585 586 static __inline u_int 587 rdr2(void) 588 { 589 u_int data; 590 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 591 return (data); 592 } 593 594 static __inline void 595 load_dr2(u_int dr2) 596 { 597 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 598 } 599 600 static __inline u_int 601 rdr3(void) 602 { 603 u_int data; 604 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 605 return (data); 606 } 607 608 static __inline void 609 load_dr3(u_int dr3) 610 { 611 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 612 } 613 614 static __inline u_int 615 rdr4(void) 616 { 617 u_int data; 618 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 619 return (data); 620 } 621 622 static __inline void 623 load_dr4(u_int dr4) 624 { 625 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 626 } 627 628 static __inline u_int 629 rdr5(void) 630 { 631 u_int data; 632 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 633 return (data); 634 } 635 636 static __inline void 637 load_dr5(u_int dr5) 638 { 639 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 640 } 641 642 static __inline u_int 643 rdr6(void) 644 { 645 u_int data; 646 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 647 return (data); 648 } 649 650 static __inline void 651 load_dr6(u_int dr6) 652 { 653 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 654 } 655 656 static __inline u_int 657 rdr7(void) 658 { 659 u_int data; 660 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 661 return (data); 662 } 663 664 static __inline void 665 load_dr7(u_int dr7) 666 { 667 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 668 } 669 670 static __inline u_char 671 read_cyrix_reg(u_char reg) 672 { 673 outb(0x22, reg); 674 return inb(0x23); 675 } 676 677 static __inline void 678 write_cyrix_reg(u_char reg, u_char data) 679 { 680 outb(0x22, reg); 681 outb(0x23, data); 682 } 683 684 static __inline register_t 685 intr_disable(void) 686 { 687 register_t eflags; 688 689 eflags = read_eflags(); 690 disable_intr(); 691 return (eflags); 692 } 693 694 static __inline void 695 intr_restore(register_t eflags) 696 { 697 write_eflags(eflags); 698 } 699 700 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 701 702 int breakpoint(void); 703 u_int bsfl(u_int mask); 704 u_int bsrl(u_int mask); 705 void clflush(u_long addr); 706 void clts(void); 707 void cpuid_count(u_int ax, u_int cx, u_int *p); 708 void disable_intr(void); 709 void do_cpuid(u_int ax, u_int *p); 710 void enable_intr(void); 711 void halt(void); 712 void ia32_pause(void); 713 u_char inb(u_int port); 714 u_int inl(u_int port); 715 void insb(u_int port, void *addr, size_t count); 716 void insl(u_int port, void *addr, size_t count); 717 void insw(u_int port, void *addr, size_t count); 718 register_t intr_disable(void); 719 void intr_restore(register_t ef); 720 void invd(void); 721 void invlpg(u_int addr); 722 void invltlb(void); 723 u_short inw(u_int port); 724 void lidt(struct region_descriptor *addr); 725 void lldt(u_short sel); 726 void load_cr0(u_int cr0); 727 void load_cr3(u_int cr3); 728 void load_cr4(u_int cr4); 729 void load_dr0(u_int dr0); 730 void load_dr1(u_int dr1); 731 void load_dr2(u_int dr2); 732 void load_dr3(u_int dr3); 733 void load_dr4(u_int dr4); 734 void load_dr5(u_int dr5); 735 void load_dr6(u_int dr6); 736 void load_dr7(u_int dr7); 737 void load_fs(u_short sel); 738 void load_gs(u_short sel); 739 void ltr(u_short sel); 740 void outb(u_int port, u_char data); 741 void outl(u_int port, u_int data); 742 void outsb(u_int port, const void *addr, size_t count); 743 void outsl(u_int port, const void *addr, size_t count); 744 void outsw(u_int port, const void *addr, size_t count); 745 void outw(u_int port, u_short data); 746 u_int rcr0(void); 747 u_int rcr2(void); 748 u_int rcr3(void); 749 u_int rcr4(void); 750 uint64_t rdmsr(u_int msr); 751 uint64_t rdpmc(u_int pmc); 752 u_int rdr0(void); 753 u_int rdr1(void); 754 u_int rdr2(void); 755 u_int rdr3(void); 756 u_int rdr4(void); 757 u_int rdr5(void); 758 u_int rdr6(void); 759 u_int rdr7(void); 760 uint64_t rdtsc(void); 761 u_char read_cyrix_reg(u_char reg); 762 u_int read_eflags(void); 763 u_int rfs(void); 764 uint64_t rgdt(void); 765 u_int rgs(void); 766 uint64_t ridt(void); 767 u_short rldt(void); 768 u_short rtr(void); 769 void wbinvd(void); 770 void write_cyrix_reg(u_char reg, u_char data); 771 void write_eflags(u_int ef); 772 void wrmsr(u_int msr, uint64_t newval); 773 774 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 775 776 void reset_dbregs(void); 777 778 #ifdef _KERNEL 779 int rdmsr_safe(u_int msr, uint64_t *val); 780 int wrmsr_safe(u_int msr, uint64_t newval); 781 #endif 782 783 #endif /* !_MACHINE_CPUFUNC_H_ */ 784