1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern void write_eflags(u_int eflags); 53 extern u_int read_eflags(void); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile uint8_t *) (va)) 59 #define readw(va) (*(volatile uint16_t *) (va)) 60 #define readl(va) (*(volatile uint32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90 } 91 92 static __inline void 93 clflush(u_long addr) 94 { 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97 } 98 99 static __inline void 100 disable_intr(void) 101 { 102 #ifdef XEN 103 xen_cli(); 104 #else 105 __asm __volatile("cli" : : : "memory"); 106 #endif 107 } 108 109 static __inline void 110 do_cpuid(u_int ax, u_int *p) 111 { 112 __asm __volatile("cpuid" 113 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 114 : "0" (ax)); 115 } 116 117 static __inline void 118 cpuid_count(u_int ax, u_int cx, u_int *p) 119 { 120 __asm __volatile("cpuid" 121 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 122 : "0" (ax), "c" (cx)); 123 } 124 125 static __inline void 126 enable_intr(void) 127 { 128 #ifdef XEN 129 xen_sti(); 130 #else 131 __asm __volatile("sti"); 132 #endif 133 } 134 135 static __inline void 136 cpu_monitor(const void *addr, int extensions, int hints) 137 { 138 __asm __volatile("monitor;" 139 : :"a" (addr), "c" (extensions), "d"(hints)); 140 } 141 142 static __inline void 143 cpu_mwait(int extensions, int hints) 144 { 145 __asm __volatile("mwait;" : :"a" (hints), "c" (extensions)); 146 } 147 148 static __inline void 149 mfence(void) 150 { 151 152 __asm __volatile("mfence" : : : "memory"); 153 } 154 155 #ifdef _KERNEL 156 157 #define HAVE_INLINE_FFS 158 159 static __inline int 160 ffs(int mask) 161 { 162 /* 163 * Note that gcc-2's builtin ffs would be used if we didn't declare 164 * this inline or turn off the builtin. The builtin is faster but 165 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 166 * versions. 167 */ 168 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 169 } 170 171 #define HAVE_INLINE_FLS 172 173 static __inline int 174 fls(int mask) 175 { 176 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 177 } 178 179 #endif /* _KERNEL */ 180 181 static __inline void 182 halt(void) 183 { 184 __asm __volatile("hlt"); 185 } 186 187 static __inline u_char 188 inb(u_int port) 189 { 190 u_char data; 191 192 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 193 return (data); 194 } 195 196 static __inline u_int 197 inl(u_int port) 198 { 199 u_int data; 200 201 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 202 return (data); 203 } 204 205 static __inline void 206 insb(u_int port, void *addr, size_t count) 207 { 208 __asm __volatile("cld; rep; insb" 209 : "+D" (addr), "+c" (count) 210 : "d" (port) 211 : "memory"); 212 } 213 214 static __inline void 215 insw(u_int port, void *addr, size_t count) 216 { 217 __asm __volatile("cld; rep; insw" 218 : "+D" (addr), "+c" (count) 219 : "d" (port) 220 : "memory"); 221 } 222 223 static __inline void 224 insl(u_int port, void *addr, size_t count) 225 { 226 __asm __volatile("cld; rep; insl" 227 : "+D" (addr), "+c" (count) 228 : "d" (port) 229 : "memory"); 230 } 231 232 static __inline void 233 invd(void) 234 { 235 __asm __volatile("invd"); 236 } 237 238 static __inline u_short 239 inw(u_int port) 240 { 241 u_short data; 242 243 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 244 return (data); 245 } 246 247 static __inline void 248 outb(u_int port, u_char data) 249 { 250 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 251 } 252 253 static __inline void 254 outl(u_int port, u_int data) 255 { 256 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 257 } 258 259 static __inline void 260 outsb(u_int port, const void *addr, size_t count) 261 { 262 __asm __volatile("cld; rep; outsb" 263 : "+S" (addr), "+c" (count) 264 : "d" (port)); 265 } 266 267 static __inline void 268 outsw(u_int port, const void *addr, size_t count) 269 { 270 __asm __volatile("cld; rep; outsw" 271 : "+S" (addr), "+c" (count) 272 : "d" (port)); 273 } 274 275 static __inline void 276 outsl(u_int port, const void *addr, size_t count) 277 { 278 __asm __volatile("cld; rep; outsl" 279 : "+S" (addr), "+c" (count) 280 : "d" (port)); 281 } 282 283 static __inline void 284 outw(u_int port, u_short data) 285 { 286 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 287 } 288 289 static __inline void 290 ia32_pause(void) 291 { 292 __asm __volatile("pause"); 293 } 294 295 static __inline u_int 296 #ifdef XEN 297 _read_eflags(void) 298 #else 299 read_eflags(void) 300 #endif 301 { 302 u_int ef; 303 304 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 305 return (ef); 306 } 307 308 static __inline uint64_t 309 rdmsr(u_int msr) 310 { 311 uint64_t rv; 312 313 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 314 return (rv); 315 } 316 317 static __inline uint64_t 318 rdpmc(u_int pmc) 319 { 320 uint64_t rv; 321 322 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 323 return (rv); 324 } 325 326 static __inline uint64_t 327 rdtsc(void) 328 { 329 uint64_t rv; 330 331 __asm __volatile("rdtsc" : "=A" (rv)); 332 return (rv); 333 } 334 335 static __inline uint32_t 336 rdtsc32(void) 337 { 338 uint32_t rv; 339 340 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 341 return (rv); 342 } 343 344 static __inline void 345 wbinvd(void) 346 { 347 __asm __volatile("wbinvd"); 348 } 349 350 static __inline void 351 #ifdef XEN 352 _write_eflags(u_int ef) 353 #else 354 write_eflags(u_int ef) 355 #endif 356 { 357 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 358 } 359 360 static __inline void 361 wrmsr(u_int msr, uint64_t newval) 362 { 363 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 364 } 365 366 static __inline void 367 load_cr0(u_int data) 368 { 369 370 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 371 } 372 373 static __inline u_int 374 rcr0(void) 375 { 376 u_int data; 377 378 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 379 return (data); 380 } 381 382 static __inline u_int 383 rcr2(void) 384 { 385 u_int data; 386 387 #ifdef XEN 388 return (xen_rcr2()); 389 #endif 390 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 391 return (data); 392 } 393 394 static __inline void 395 load_cr3(u_int data) 396 { 397 #ifdef XEN 398 xen_load_cr3(data); 399 #else 400 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 401 #endif 402 } 403 404 static __inline u_int 405 rcr3(void) 406 { 407 u_int data; 408 409 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 410 return (data); 411 } 412 413 static __inline void 414 load_cr4(u_int data) 415 { 416 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 417 } 418 419 static __inline u_int 420 rcr4(void) 421 { 422 u_int data; 423 424 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 425 return (data); 426 } 427 428 /* 429 * Global TLB flush (except for thise for pages marked PG_G) 430 */ 431 static __inline void 432 invltlb(void) 433 { 434 #ifdef XEN 435 xen_tlb_flush(); 436 #else 437 load_cr3(rcr3()); 438 #endif 439 } 440 441 /* 442 * TLB flush for an individual page (even if it has PG_G). 443 * Only works on 486+ CPUs (i386 does not have PG_G). 444 */ 445 static __inline void 446 invlpg(u_int addr) 447 { 448 449 #ifdef XEN 450 xen_invlpg(addr); 451 #else 452 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 453 #endif 454 } 455 456 static __inline u_short 457 rfs(void) 458 { 459 u_short sel; 460 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 461 return (sel); 462 } 463 464 static __inline uint64_t 465 rgdt(void) 466 { 467 uint64_t gdtr; 468 __asm __volatile("sgdt %0" : "=m" (gdtr)); 469 return (gdtr); 470 } 471 472 static __inline u_short 473 rgs(void) 474 { 475 u_short sel; 476 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 477 return (sel); 478 } 479 480 static __inline uint64_t 481 ridt(void) 482 { 483 uint64_t idtr; 484 __asm __volatile("sidt %0" : "=m" (idtr)); 485 return (idtr); 486 } 487 488 static __inline u_short 489 rldt(void) 490 { 491 u_short ldtr; 492 __asm __volatile("sldt %0" : "=g" (ldtr)); 493 return (ldtr); 494 } 495 496 static __inline u_short 497 rss(void) 498 { 499 u_short sel; 500 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 501 return (sel); 502 } 503 504 static __inline u_short 505 rtr(void) 506 { 507 u_short tr; 508 __asm __volatile("str %0" : "=g" (tr)); 509 return (tr); 510 } 511 512 static __inline void 513 load_fs(u_short sel) 514 { 515 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 516 } 517 518 static __inline void 519 load_gs(u_short sel) 520 { 521 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 522 } 523 524 static __inline void 525 lidt(struct region_descriptor *addr) 526 { 527 __asm __volatile("lidt (%0)" : : "r" (addr)); 528 } 529 530 static __inline void 531 lldt(u_short sel) 532 { 533 __asm __volatile("lldt %0" : : "r" (sel)); 534 } 535 536 static __inline void 537 ltr(u_short sel) 538 { 539 __asm __volatile("ltr %0" : : "r" (sel)); 540 } 541 542 static __inline u_int 543 rdr0(void) 544 { 545 u_int data; 546 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 547 return (data); 548 } 549 550 static __inline void 551 load_dr0(u_int dr0) 552 { 553 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 554 } 555 556 static __inline u_int 557 rdr1(void) 558 { 559 u_int data; 560 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 561 return (data); 562 } 563 564 static __inline void 565 load_dr1(u_int dr1) 566 { 567 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 568 } 569 570 static __inline u_int 571 rdr2(void) 572 { 573 u_int data; 574 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 575 return (data); 576 } 577 578 static __inline void 579 load_dr2(u_int dr2) 580 { 581 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 582 } 583 584 static __inline u_int 585 rdr3(void) 586 { 587 u_int data; 588 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 589 return (data); 590 } 591 592 static __inline void 593 load_dr3(u_int dr3) 594 { 595 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 596 } 597 598 static __inline u_int 599 rdr4(void) 600 { 601 u_int data; 602 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 603 return (data); 604 } 605 606 static __inline void 607 load_dr4(u_int dr4) 608 { 609 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 610 } 611 612 static __inline u_int 613 rdr5(void) 614 { 615 u_int data; 616 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 617 return (data); 618 } 619 620 static __inline void 621 load_dr5(u_int dr5) 622 { 623 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 624 } 625 626 static __inline u_int 627 rdr6(void) 628 { 629 u_int data; 630 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 631 return (data); 632 } 633 634 static __inline void 635 load_dr6(u_int dr6) 636 { 637 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 638 } 639 640 static __inline u_int 641 rdr7(void) 642 { 643 u_int data; 644 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 645 return (data); 646 } 647 648 static __inline void 649 load_dr7(u_int dr7) 650 { 651 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 652 } 653 654 static __inline u_char 655 read_cyrix_reg(u_char reg) 656 { 657 outb(0x22, reg); 658 return inb(0x23); 659 } 660 661 static __inline void 662 write_cyrix_reg(u_char reg, u_char data) 663 { 664 outb(0x22, reg); 665 outb(0x23, data); 666 } 667 668 static __inline register_t 669 intr_disable(void) 670 { 671 register_t eflags; 672 673 eflags = read_eflags(); 674 disable_intr(); 675 return (eflags); 676 } 677 678 static __inline void 679 intr_restore(register_t eflags) 680 { 681 write_eflags(eflags); 682 } 683 684 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 685 686 int breakpoint(void); 687 u_int bsfl(u_int mask); 688 u_int bsrl(u_int mask); 689 void disable_intr(void); 690 void do_cpuid(u_int ax, u_int *p); 691 void enable_intr(void); 692 void halt(void); 693 void ia32_pause(void); 694 u_char inb(u_int port); 695 u_int inl(u_int port); 696 void insb(u_int port, void *addr, size_t count); 697 void insl(u_int port, void *addr, size_t count); 698 void insw(u_int port, void *addr, size_t count); 699 register_t intr_disable(void); 700 void intr_restore(register_t ef); 701 void invd(void); 702 void invlpg(u_int addr); 703 void invltlb(void); 704 u_short inw(u_int port); 705 void lidt(struct region_descriptor *addr); 706 void lldt(u_short sel); 707 void load_cr0(u_int cr0); 708 void load_cr3(u_int cr3); 709 void load_cr4(u_int cr4); 710 void load_dr0(u_int dr0); 711 void load_dr1(u_int dr1); 712 void load_dr2(u_int dr2); 713 void load_dr3(u_int dr3); 714 void load_dr4(u_int dr4); 715 void load_dr5(u_int dr5); 716 void load_dr6(u_int dr6); 717 void load_dr7(u_int dr7); 718 void load_fs(u_short sel); 719 void load_gs(u_short sel); 720 void ltr(u_short sel); 721 void outb(u_int port, u_char data); 722 void outl(u_int port, u_int data); 723 void outsb(u_int port, const void *addr, size_t count); 724 void outsl(u_int port, const void *addr, size_t count); 725 void outsw(u_int port, const void *addr, size_t count); 726 void outw(u_int port, u_short data); 727 u_int rcr0(void); 728 u_int rcr2(void); 729 u_int rcr3(void); 730 u_int rcr4(void); 731 uint64_t rdmsr(u_int msr); 732 uint64_t rdpmc(u_int pmc); 733 u_int rdr0(void); 734 u_int rdr1(void); 735 u_int rdr2(void); 736 u_int rdr3(void); 737 u_int rdr4(void); 738 u_int rdr5(void); 739 u_int rdr6(void); 740 u_int rdr7(void); 741 uint64_t rdtsc(void); 742 u_char read_cyrix_reg(u_char reg); 743 u_int read_eflags(void); 744 u_int rfs(void); 745 uint64_t rgdt(void); 746 u_int rgs(void); 747 uint64_t ridt(void); 748 u_short rldt(void); 749 u_short rtr(void); 750 void wbinvd(void); 751 void write_cyrix_reg(u_char reg, u_char data); 752 void write_eflags(u_int ef); 753 void wrmsr(u_int msr, uint64_t newval); 754 755 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 756 757 void reset_dbregs(void); 758 759 #ifdef _KERNEL 760 int rdmsr_safe(u_int msr, uint64_t *val); 761 int wrmsr_safe(u_int msr, uint64_t newval); 762 #endif 763 764 #endif /* !_MACHINE_CPUFUNC_H_ */ 765