1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern int xen_save_and_cli(void); 53 extern void xen_restore_flags(u_int eflags); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile u_int8_t *) (va)) 59 #define readw(va) (*(volatile u_int16_t *) (va)) 60 #define readl(va) (*(volatile u_int32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 89 return (result); 90 } 91 92 static __inline void 93 disable_intr(void) 94 { 95 #ifdef XEN 96 xen_cli(); 97 #else 98 __asm __volatile("cli" : : : "memory"); 99 #endif 100 } 101 102 static __inline void 103 do_cpuid(u_int ax, u_int *p) 104 { 105 __asm __volatile("cpuid" 106 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 107 : "0" (ax)); 108 } 109 110 static __inline void 111 cpuid_count(u_int ax, u_int cx, u_int *p) 112 { 113 __asm __volatile("cpuid" 114 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 115 : "0" (ax), "c" (cx)); 116 } 117 118 static __inline void 119 enable_intr(void) 120 { 121 #ifdef XEN 122 xen_sti(); 123 #else 124 __asm __volatile("sti"); 125 #endif 126 } 127 128 static inline void 129 cpu_monitor(const void *addr, int extensions, int hints) 130 { 131 __asm __volatile("monitor;" 132 : :"a" (addr), "c" (extensions), "d"(hints)); 133 } 134 135 static inline void 136 cpu_mwait(int extensions, int hints) 137 { 138 __asm __volatile("mwait;" : :"a" (hints), "c" (extensions)); 139 } 140 141 #ifdef _KERNEL 142 143 #define HAVE_INLINE_FFS 144 145 static __inline int 146 ffs(int mask) 147 { 148 /* 149 * Note that gcc-2's builtin ffs would be used if we didn't declare 150 * this inline or turn off the builtin. The builtin is faster but 151 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 152 * versions. 153 */ 154 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 155 } 156 157 #define HAVE_INLINE_FLS 158 159 static __inline int 160 fls(int mask) 161 { 162 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 163 } 164 165 #endif /* _KERNEL */ 166 167 static __inline void 168 halt(void) 169 { 170 __asm __volatile("hlt"); 171 } 172 173 #if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3 174 175 #define inb(port) inbv(port) 176 #define outb(port, data) outbv(port, data) 177 178 #else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */ 179 180 /* 181 * The following complications are to get around gcc not having a 182 * constraint letter for the range 0..255. We still put "d" in the 183 * constraint because "i" isn't a valid constraint when the port 184 * isn't constant. This only matters for -O0 because otherwise 185 * the non-working version gets optimized away. 186 * 187 * Use an expression-statement instead of a conditional expression 188 * because gcc-2.6.0 would promote the operands of the conditional 189 * and produce poor code for "if ((inb(var) & const1) == const2)". 190 * 191 * The unnecessary test `(port) < 0x10000' is to generate a warning if 192 * the `port' has type u_short or smaller. Such types are pessimal. 193 * This actually only works for signed types. The range check is 194 * careful to avoid generating warnings. 195 */ 196 #define inb(port) __extension__ ({ \ 197 u_char _data; \ 198 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 199 && (port) < 0x10000) \ 200 _data = inbc(port); \ 201 else \ 202 _data = inbv(port); \ 203 _data; }) 204 205 #define outb(port, data) ( \ 206 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 207 && (port) < 0x10000 \ 208 ? outbc(port, data) : outbv(port, data)) 209 210 static __inline u_char 211 inbc(u_int port) 212 { 213 u_char data; 214 215 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 216 return (data); 217 } 218 219 static __inline void 220 outbc(u_int port, u_char data) 221 { 222 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 223 } 224 225 #endif /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3*/ 226 227 static __inline u_char 228 inbv(u_int port) 229 { 230 u_char data; 231 /* 232 * We use %%dx and not %1 here because i/o is done at %dx and not at 233 * %edx, while gcc generates inferior code (movw instead of movl) 234 * if we tell it to load (u_short) port. 235 */ 236 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 237 return (data); 238 } 239 240 static __inline u_int 241 inl(u_int port) 242 { 243 u_int data; 244 245 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 246 return (data); 247 } 248 249 static __inline void 250 insb(u_int port, void *addr, size_t cnt) 251 { 252 __asm __volatile("cld; rep; insb" 253 : "+D" (addr), "+c" (cnt) 254 : "d" (port) 255 : "memory"); 256 } 257 258 static __inline void 259 insw(u_int port, void *addr, size_t cnt) 260 { 261 __asm __volatile("cld; rep; insw" 262 : "+D" (addr), "+c" (cnt) 263 : "d" (port) 264 : "memory"); 265 } 266 267 static __inline void 268 insl(u_int port, void *addr, size_t cnt) 269 { 270 __asm __volatile("cld; rep; insl" 271 : "+D" (addr), "+c" (cnt) 272 : "d" (port) 273 : "memory"); 274 } 275 276 static __inline void 277 invd(void) 278 { 279 __asm __volatile("invd"); 280 } 281 282 static __inline u_short 283 inw(u_int port) 284 { 285 u_short data; 286 287 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 288 return (data); 289 } 290 291 static __inline void 292 outbv(u_int port, u_char data) 293 { 294 u_char al; 295 /* 296 * Use an unnecessary assignment to help gcc's register allocator. 297 * This make a large difference for gcc-1.40 and a tiny difference 298 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 299 * best results. gcc-2.6.0 can't handle this. 300 */ 301 al = data; 302 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 303 } 304 305 static __inline void 306 outl(u_int port, u_int data) 307 { 308 /* 309 * outl() and outw() aren't used much so we haven't looked at 310 * possible micro-optimizations such as the unnecessary 311 * assignment for them. 312 */ 313 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 314 } 315 316 static __inline void 317 outsb(u_int port, const void *addr, size_t cnt) 318 { 319 __asm __volatile("cld; rep; outsb" 320 : "+S" (addr), "+c" (cnt) 321 : "d" (port)); 322 } 323 324 static __inline void 325 outsw(u_int port, const void *addr, size_t cnt) 326 { 327 __asm __volatile("cld; rep; outsw" 328 : "+S" (addr), "+c" (cnt) 329 : "d" (port)); 330 } 331 332 static __inline void 333 outsl(u_int port, const void *addr, size_t cnt) 334 { 335 __asm __volatile("cld; rep; outsl" 336 : "+S" (addr), "+c" (cnt) 337 : "d" (port)); 338 } 339 340 static __inline void 341 outw(u_int port, u_short data) 342 { 343 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 344 } 345 346 static __inline void 347 ia32_pause(void) 348 { 349 __asm __volatile("pause"); 350 } 351 352 static __inline u_int 353 read_eflags(void) 354 { 355 u_int ef; 356 357 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 358 return (ef); 359 } 360 361 static __inline uint64_t 362 rdmsr(u_int msr) 363 { 364 uint64_t rv; 365 366 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 367 return (rv); 368 } 369 370 static __inline uint64_t 371 rdpmc(u_int pmc) 372 { 373 uint64_t rv; 374 375 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 376 return (rv); 377 } 378 379 static __inline uint64_t 380 rdtsc(void) 381 { 382 uint64_t rv; 383 384 __asm __volatile("rdtsc" : "=A" (rv)); 385 return (rv); 386 } 387 388 static __inline void 389 wbinvd(void) 390 { 391 __asm __volatile("wbinvd"); 392 } 393 394 static __inline void 395 write_eflags(u_int ef) 396 { 397 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 398 } 399 400 static __inline void 401 wrmsr(u_int msr, uint64_t newval) 402 { 403 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 404 } 405 406 static __inline void 407 load_cr0(u_int data) 408 { 409 410 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 411 } 412 413 static __inline u_int 414 rcr0(void) 415 { 416 u_int data; 417 418 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 419 return (data); 420 } 421 422 static __inline u_int 423 rcr2(void) 424 { 425 u_int data; 426 427 #ifdef XEN 428 return (xen_rcr2()); 429 #endif 430 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 431 return (data); 432 } 433 434 static __inline void 435 load_cr3(u_int data) 436 { 437 #ifdef XEN 438 xen_load_cr3(data); 439 #else 440 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 441 #endif 442 } 443 444 static __inline u_int 445 rcr3(void) 446 { 447 u_int data; 448 449 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 450 return (data); 451 } 452 453 static __inline void 454 load_cr4(u_int data) 455 { 456 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 457 } 458 459 static __inline u_int 460 rcr4(void) 461 { 462 u_int data; 463 464 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 465 return (data); 466 } 467 468 /* 469 * Global TLB flush (except for thise for pages marked PG_G) 470 */ 471 static __inline void 472 invltlb(void) 473 { 474 #ifdef XEN 475 xen_tlb_flush(); 476 #else 477 load_cr3(rcr3()); 478 #endif 479 } 480 481 /* 482 * TLB flush for an individual page (even if it has PG_G). 483 * Only works on 486+ CPUs (i386 does not have PG_G). 484 */ 485 static __inline void 486 invlpg(u_int addr) 487 { 488 489 #ifdef XEN 490 xen_invlpg(addr); 491 #else 492 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 493 #endif 494 } 495 496 static __inline u_int 497 rfs(void) 498 { 499 u_int sel; 500 __asm __volatile("mov %%fs,%0" : "=rm" (sel)); 501 return (sel); 502 } 503 504 static __inline uint64_t 505 rgdt(void) 506 { 507 uint64_t gdtr; 508 __asm __volatile("sgdt %0" : "=m" (gdtr)); 509 return (gdtr); 510 } 511 512 static __inline u_int 513 rgs(void) 514 { 515 u_int sel; 516 __asm __volatile("mov %%gs,%0" : "=rm" (sel)); 517 return (sel); 518 } 519 520 static __inline uint64_t 521 ridt(void) 522 { 523 uint64_t idtr; 524 __asm __volatile("sidt %0" : "=m" (idtr)); 525 return (idtr); 526 } 527 528 static __inline u_short 529 rldt(void) 530 { 531 u_short ldtr; 532 __asm __volatile("sldt %0" : "=g" (ldtr)); 533 return (ldtr); 534 } 535 536 static __inline u_int 537 rss(void) 538 { 539 u_int sel; 540 __asm __volatile("mov %%ss,%0" : "=rm" (sel)); 541 return (sel); 542 } 543 544 static __inline u_short 545 rtr(void) 546 { 547 u_short tr; 548 __asm __volatile("str %0" : "=g" (tr)); 549 return (tr); 550 } 551 552 static __inline void 553 load_fs(u_int sel) 554 { 555 __asm __volatile("mov %0,%%fs" : : "rm" (sel)); 556 } 557 558 static __inline void 559 load_gs(u_int sel) 560 { 561 __asm __volatile("mov %0,%%gs" : : "rm" (sel)); 562 } 563 564 static __inline void 565 lidt(struct region_descriptor *addr) 566 { 567 __asm __volatile("lidt (%0)" : : "r" (addr)); 568 } 569 570 static __inline void 571 lldt(u_short sel) 572 { 573 __asm __volatile("lldt %0" : : "r" (sel)); 574 } 575 576 static __inline void 577 ltr(u_short sel) 578 { 579 __asm __volatile("ltr %0" : : "r" (sel)); 580 } 581 582 static __inline u_int 583 rdr0(void) 584 { 585 u_int data; 586 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 587 return (data); 588 } 589 590 static __inline void 591 load_dr0(u_int dr0) 592 { 593 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 594 } 595 596 static __inline u_int 597 rdr1(void) 598 { 599 u_int data; 600 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 601 return (data); 602 } 603 604 static __inline void 605 load_dr1(u_int dr1) 606 { 607 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 608 } 609 610 static __inline u_int 611 rdr2(void) 612 { 613 u_int data; 614 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 615 return (data); 616 } 617 618 static __inline void 619 load_dr2(u_int dr2) 620 { 621 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 622 } 623 624 static __inline u_int 625 rdr3(void) 626 { 627 u_int data; 628 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 629 return (data); 630 } 631 632 static __inline void 633 load_dr3(u_int dr3) 634 { 635 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 636 } 637 638 static __inline u_int 639 rdr4(void) 640 { 641 u_int data; 642 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 643 return (data); 644 } 645 646 static __inline void 647 load_dr4(u_int dr4) 648 { 649 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 650 } 651 652 static __inline u_int 653 rdr5(void) 654 { 655 u_int data; 656 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 657 return (data); 658 } 659 660 static __inline void 661 load_dr5(u_int dr5) 662 { 663 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 664 } 665 666 static __inline u_int 667 rdr6(void) 668 { 669 u_int data; 670 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 671 return (data); 672 } 673 674 static __inline void 675 load_dr6(u_int dr6) 676 { 677 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 678 } 679 680 static __inline u_int 681 rdr7(void) 682 { 683 u_int data; 684 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 685 return (data); 686 } 687 688 static __inline void 689 load_dr7(u_int dr7) 690 { 691 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 692 } 693 694 static __inline register_t 695 intr_disable(void) 696 { 697 register_t eflags; 698 699 #ifdef XEN 700 eflags = xen_save_and_cli(); 701 #else 702 eflags = read_eflags(); 703 disable_intr(); 704 #endif 705 return (eflags); 706 } 707 708 static __inline void 709 intr_restore(register_t eflags) 710 { 711 #ifdef XEN 712 xen_restore_flags(eflags); 713 #else 714 write_eflags(eflags); 715 #endif 716 } 717 718 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 719 720 int breakpoint(void); 721 u_int bsfl(u_int mask); 722 u_int bsrl(u_int mask); 723 void disable_intr(void); 724 void do_cpuid(u_int ax, u_int *p); 725 void enable_intr(void); 726 void halt(void); 727 void ia32_pause(void); 728 u_char inb(u_int port); 729 u_int inl(u_int port); 730 void insb(u_int port, void *addr, size_t cnt); 731 void insl(u_int port, void *addr, size_t cnt); 732 void insw(u_int port, void *addr, size_t cnt); 733 register_t intr_disable(void); 734 void intr_restore(register_t ef); 735 void invd(void); 736 void invlpg(u_int addr); 737 void invltlb(void); 738 u_short inw(u_int port); 739 void lidt(struct region_descriptor *addr); 740 void lldt(u_short sel); 741 void load_cr0(u_int cr0); 742 void load_cr3(u_int cr3); 743 void load_cr4(u_int cr4); 744 void load_dr0(u_int dr0); 745 void load_dr1(u_int dr1); 746 void load_dr2(u_int dr2); 747 void load_dr3(u_int dr3); 748 void load_dr4(u_int dr4); 749 void load_dr5(u_int dr5); 750 void load_dr6(u_int dr6); 751 void load_dr7(u_int dr7); 752 void load_fs(u_int sel); 753 void load_gs(u_int sel); 754 void ltr(u_short sel); 755 void outb(u_int port, u_char data); 756 void outl(u_int port, u_int data); 757 void outsb(u_int port, const void *addr, size_t cnt); 758 void outsl(u_int port, const void *addr, size_t cnt); 759 void outsw(u_int port, const void *addr, size_t cnt); 760 void outw(u_int port, u_short data); 761 u_int rcr0(void); 762 u_int rcr2(void); 763 u_int rcr3(void); 764 u_int rcr4(void); 765 uint64_t rdmsr(u_int msr); 766 uint64_t rdpmc(u_int pmc); 767 u_int rdr0(void); 768 u_int rdr1(void); 769 u_int rdr2(void); 770 u_int rdr3(void); 771 u_int rdr4(void); 772 u_int rdr5(void); 773 u_int rdr6(void); 774 u_int rdr7(void); 775 uint64_t rdtsc(void); 776 u_int read_eflags(void); 777 u_int rfs(void); 778 uint64_t rgdt(void); 779 u_int rgs(void); 780 uint64_t ridt(void); 781 u_short rldt(void); 782 u_short rtr(void); 783 void wbinvd(void); 784 void write_eflags(u_int ef); 785 void wrmsr(u_int msr, uint64_t newval); 786 787 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 788 789 void reset_dbregs(void); 790 791 #ifdef _KERNEL 792 int rdmsr_safe(u_int msr, uint64_t *val); 793 int wrmsr_safe(u_int msr, uint64_t newval); 794 #endif 795 796 #endif /* !_MACHINE_CPUFUNC_H_ */ 797