1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern void write_eflags(u_int eflags); 53 extern u_int read_eflags(void); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile uint8_t *) (va)) 59 #define readw(va) (*(volatile uint16_t *) (va)) 60 #define readl(va) (*(volatile uint32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90 } 91 92 static __inline void 93 clflush(u_long addr) 94 { 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97 } 98 99 static __inline void 100 disable_intr(void) 101 { 102 #ifdef XEN 103 xen_cli(); 104 #else 105 __asm __volatile("cli" : : : "memory"); 106 #endif 107 } 108 109 static __inline void 110 do_cpuid(u_int ax, u_int *p) 111 { 112 __asm __volatile("cpuid" 113 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 114 : "0" (ax)); 115 } 116 117 static __inline void 118 cpuid_count(u_int ax, u_int cx, u_int *p) 119 { 120 __asm __volatile("cpuid" 121 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 122 : "0" (ax), "c" (cx)); 123 } 124 125 static __inline void 126 enable_intr(void) 127 { 128 #ifdef XEN 129 xen_sti(); 130 #else 131 __asm __volatile("sti"); 132 #endif 133 } 134 135 static __inline void 136 cpu_monitor(const void *addr, u_long extensions, u_int hints) 137 { 138 139 __asm __volatile("monitor" 140 : : "a" (addr), "c" (extensions), "d" (hints)); 141 } 142 143 static __inline void 144 cpu_mwait(u_long extensions, u_int hints) 145 { 146 147 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 148 } 149 150 static __inline void 151 mfence(void) 152 { 153 154 __asm __volatile("mfence" : : : "memory"); 155 } 156 157 #ifdef _KERNEL 158 159 #define HAVE_INLINE_FFS 160 161 static __inline int 162 ffs(int mask) 163 { 164 /* 165 * Note that gcc-2's builtin ffs would be used if we didn't declare 166 * this inline or turn off the builtin. The builtin is faster but 167 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 168 * versions. 169 */ 170 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 171 } 172 173 #define HAVE_INLINE_FLS 174 175 static __inline int 176 fls(int mask) 177 { 178 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 179 } 180 181 #endif /* _KERNEL */ 182 183 static __inline void 184 halt(void) 185 { 186 __asm __volatile("hlt"); 187 } 188 189 static __inline u_char 190 inb(u_int port) 191 { 192 u_char data; 193 194 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 195 return (data); 196 } 197 198 static __inline u_int 199 inl(u_int port) 200 { 201 u_int data; 202 203 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 204 return (data); 205 } 206 207 static __inline void 208 insb(u_int port, void *addr, size_t count) 209 { 210 __asm __volatile("cld; rep; insb" 211 : "+D" (addr), "+c" (count) 212 : "d" (port) 213 : "memory"); 214 } 215 216 static __inline void 217 insw(u_int port, void *addr, size_t count) 218 { 219 __asm __volatile("cld; rep; insw" 220 : "+D" (addr), "+c" (count) 221 : "d" (port) 222 : "memory"); 223 } 224 225 static __inline void 226 insl(u_int port, void *addr, size_t count) 227 { 228 __asm __volatile("cld; rep; insl" 229 : "+D" (addr), "+c" (count) 230 : "d" (port) 231 : "memory"); 232 } 233 234 static __inline void 235 invd(void) 236 { 237 __asm __volatile("invd"); 238 } 239 240 static __inline u_short 241 inw(u_int port) 242 { 243 u_short data; 244 245 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 246 return (data); 247 } 248 249 static __inline void 250 outb(u_int port, u_char data) 251 { 252 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 253 } 254 255 static __inline void 256 outl(u_int port, u_int data) 257 { 258 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 259 } 260 261 static __inline void 262 outsb(u_int port, const void *addr, size_t count) 263 { 264 __asm __volatile("cld; rep; outsb" 265 : "+S" (addr), "+c" (count) 266 : "d" (port)); 267 } 268 269 static __inline void 270 outsw(u_int port, const void *addr, size_t count) 271 { 272 __asm __volatile("cld; rep; outsw" 273 : "+S" (addr), "+c" (count) 274 : "d" (port)); 275 } 276 277 static __inline void 278 outsl(u_int port, const void *addr, size_t count) 279 { 280 __asm __volatile("cld; rep; outsl" 281 : "+S" (addr), "+c" (count) 282 : "d" (port)); 283 } 284 285 static __inline void 286 outw(u_int port, u_short data) 287 { 288 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 289 } 290 291 static __inline void 292 ia32_pause(void) 293 { 294 __asm __volatile("pause"); 295 } 296 297 static __inline u_int 298 #ifdef XEN 299 _read_eflags(void) 300 #else 301 read_eflags(void) 302 #endif 303 { 304 u_int ef; 305 306 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 307 return (ef); 308 } 309 310 static __inline uint64_t 311 rdmsr(u_int msr) 312 { 313 uint64_t rv; 314 315 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 316 return (rv); 317 } 318 319 static __inline uint64_t 320 rdpmc(u_int pmc) 321 { 322 uint64_t rv; 323 324 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 325 return (rv); 326 } 327 328 static __inline uint64_t 329 rdtsc(void) 330 { 331 uint64_t rv; 332 333 __asm __volatile("rdtsc" : "=A" (rv)); 334 return (rv); 335 } 336 337 static __inline uint32_t 338 rdtsc32(void) 339 { 340 uint32_t rv; 341 342 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 343 return (rv); 344 } 345 346 static __inline void 347 wbinvd(void) 348 { 349 __asm __volatile("wbinvd"); 350 } 351 352 static __inline void 353 #ifdef XEN 354 _write_eflags(u_int ef) 355 #else 356 write_eflags(u_int ef) 357 #endif 358 { 359 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 360 } 361 362 static __inline void 363 wrmsr(u_int msr, uint64_t newval) 364 { 365 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 366 } 367 368 static __inline void 369 load_cr0(u_int data) 370 { 371 372 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 373 } 374 375 static __inline u_int 376 rcr0(void) 377 { 378 u_int data; 379 380 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 381 return (data); 382 } 383 384 static __inline u_int 385 rcr2(void) 386 { 387 u_int data; 388 389 #ifdef XEN 390 return (xen_rcr2()); 391 #endif 392 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 393 return (data); 394 } 395 396 static __inline void 397 load_cr3(u_int data) 398 { 399 #ifdef XEN 400 xen_load_cr3(data); 401 #else 402 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 403 #endif 404 } 405 406 static __inline u_int 407 rcr3(void) 408 { 409 u_int data; 410 411 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 412 return (data); 413 } 414 415 static __inline void 416 load_cr4(u_int data) 417 { 418 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 419 } 420 421 static __inline u_int 422 rcr4(void) 423 { 424 u_int data; 425 426 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 427 return (data); 428 } 429 430 /* 431 * Global TLB flush (except for thise for pages marked PG_G) 432 */ 433 static __inline void 434 invltlb(void) 435 { 436 #ifdef XEN 437 xen_tlb_flush(); 438 #else 439 load_cr3(rcr3()); 440 #endif 441 } 442 443 /* 444 * TLB flush for an individual page (even if it has PG_G). 445 * Only works on 486+ CPUs (i386 does not have PG_G). 446 */ 447 static __inline void 448 invlpg(u_int addr) 449 { 450 451 #ifdef XEN 452 xen_invlpg(addr); 453 #else 454 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 455 #endif 456 } 457 458 static __inline u_short 459 rfs(void) 460 { 461 u_short sel; 462 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 463 return (sel); 464 } 465 466 static __inline uint64_t 467 rgdt(void) 468 { 469 uint64_t gdtr; 470 __asm __volatile("sgdt %0" : "=m" (gdtr)); 471 return (gdtr); 472 } 473 474 static __inline u_short 475 rgs(void) 476 { 477 u_short sel; 478 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 479 return (sel); 480 } 481 482 static __inline uint64_t 483 ridt(void) 484 { 485 uint64_t idtr; 486 __asm __volatile("sidt %0" : "=m" (idtr)); 487 return (idtr); 488 } 489 490 static __inline u_short 491 rldt(void) 492 { 493 u_short ldtr; 494 __asm __volatile("sldt %0" : "=g" (ldtr)); 495 return (ldtr); 496 } 497 498 static __inline u_short 499 rss(void) 500 { 501 u_short sel; 502 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 503 return (sel); 504 } 505 506 static __inline u_short 507 rtr(void) 508 { 509 u_short tr; 510 __asm __volatile("str %0" : "=g" (tr)); 511 return (tr); 512 } 513 514 static __inline void 515 load_fs(u_short sel) 516 { 517 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 518 } 519 520 static __inline void 521 load_gs(u_short sel) 522 { 523 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 524 } 525 526 static __inline void 527 lidt(struct region_descriptor *addr) 528 { 529 __asm __volatile("lidt (%0)" : : "r" (addr)); 530 } 531 532 static __inline void 533 lldt(u_short sel) 534 { 535 __asm __volatile("lldt %0" : : "r" (sel)); 536 } 537 538 static __inline void 539 ltr(u_short sel) 540 { 541 __asm __volatile("ltr %0" : : "r" (sel)); 542 } 543 544 static __inline u_int 545 rdr0(void) 546 { 547 u_int data; 548 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 549 return (data); 550 } 551 552 static __inline void 553 load_dr0(u_int dr0) 554 { 555 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 556 } 557 558 static __inline u_int 559 rdr1(void) 560 { 561 u_int data; 562 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 563 return (data); 564 } 565 566 static __inline void 567 load_dr1(u_int dr1) 568 { 569 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 570 } 571 572 static __inline u_int 573 rdr2(void) 574 { 575 u_int data; 576 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 577 return (data); 578 } 579 580 static __inline void 581 load_dr2(u_int dr2) 582 { 583 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 584 } 585 586 static __inline u_int 587 rdr3(void) 588 { 589 u_int data; 590 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 591 return (data); 592 } 593 594 static __inline void 595 load_dr3(u_int dr3) 596 { 597 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 598 } 599 600 static __inline u_int 601 rdr4(void) 602 { 603 u_int data; 604 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 605 return (data); 606 } 607 608 static __inline void 609 load_dr4(u_int dr4) 610 { 611 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 612 } 613 614 static __inline u_int 615 rdr5(void) 616 { 617 u_int data; 618 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 619 return (data); 620 } 621 622 static __inline void 623 load_dr5(u_int dr5) 624 { 625 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 626 } 627 628 static __inline u_int 629 rdr6(void) 630 { 631 u_int data; 632 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 633 return (data); 634 } 635 636 static __inline void 637 load_dr6(u_int dr6) 638 { 639 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 640 } 641 642 static __inline u_int 643 rdr7(void) 644 { 645 u_int data; 646 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 647 return (data); 648 } 649 650 static __inline void 651 load_dr7(u_int dr7) 652 { 653 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 654 } 655 656 static __inline u_char 657 read_cyrix_reg(u_char reg) 658 { 659 outb(0x22, reg); 660 return inb(0x23); 661 } 662 663 static __inline void 664 write_cyrix_reg(u_char reg, u_char data) 665 { 666 outb(0x22, reg); 667 outb(0x23, data); 668 } 669 670 static __inline register_t 671 intr_disable(void) 672 { 673 register_t eflags; 674 675 eflags = read_eflags(); 676 disable_intr(); 677 return (eflags); 678 } 679 680 static __inline void 681 intr_restore(register_t eflags) 682 { 683 write_eflags(eflags); 684 } 685 686 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 687 688 int breakpoint(void); 689 u_int bsfl(u_int mask); 690 u_int bsrl(u_int mask); 691 void disable_intr(void); 692 void do_cpuid(u_int ax, u_int *p); 693 void enable_intr(void); 694 void halt(void); 695 void ia32_pause(void); 696 u_char inb(u_int port); 697 u_int inl(u_int port); 698 void insb(u_int port, void *addr, size_t count); 699 void insl(u_int port, void *addr, size_t count); 700 void insw(u_int port, void *addr, size_t count); 701 register_t intr_disable(void); 702 void intr_restore(register_t ef); 703 void invd(void); 704 void invlpg(u_int addr); 705 void invltlb(void); 706 u_short inw(u_int port); 707 void lidt(struct region_descriptor *addr); 708 void lldt(u_short sel); 709 void load_cr0(u_int cr0); 710 void load_cr3(u_int cr3); 711 void load_cr4(u_int cr4); 712 void load_dr0(u_int dr0); 713 void load_dr1(u_int dr1); 714 void load_dr2(u_int dr2); 715 void load_dr3(u_int dr3); 716 void load_dr4(u_int dr4); 717 void load_dr5(u_int dr5); 718 void load_dr6(u_int dr6); 719 void load_dr7(u_int dr7); 720 void load_fs(u_short sel); 721 void load_gs(u_short sel); 722 void ltr(u_short sel); 723 void outb(u_int port, u_char data); 724 void outl(u_int port, u_int data); 725 void outsb(u_int port, const void *addr, size_t count); 726 void outsl(u_int port, const void *addr, size_t count); 727 void outsw(u_int port, const void *addr, size_t count); 728 void outw(u_int port, u_short data); 729 u_int rcr0(void); 730 u_int rcr2(void); 731 u_int rcr3(void); 732 u_int rcr4(void); 733 uint64_t rdmsr(u_int msr); 734 uint64_t rdpmc(u_int pmc); 735 u_int rdr0(void); 736 u_int rdr1(void); 737 u_int rdr2(void); 738 u_int rdr3(void); 739 u_int rdr4(void); 740 u_int rdr5(void); 741 u_int rdr6(void); 742 u_int rdr7(void); 743 uint64_t rdtsc(void); 744 u_char read_cyrix_reg(u_char reg); 745 u_int read_eflags(void); 746 u_int rfs(void); 747 uint64_t rgdt(void); 748 u_int rgs(void); 749 uint64_t ridt(void); 750 u_short rldt(void); 751 u_short rtr(void); 752 void wbinvd(void); 753 void write_cyrix_reg(u_char reg, u_char data); 754 void write_eflags(u_int ef); 755 void wrmsr(u_int msr, uint64_t newval); 756 757 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 758 759 void reset_dbregs(void); 760 761 #ifdef _KERNEL 762 int rdmsr_safe(u_int msr, uint64_t *val); 763 int wrmsr_safe(u_int msr, uint64_t newval); 764 #endif 765 766 #endif /* !_MACHINE_CPUFUNC_H_ */ 767