1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 struct region_descriptor; 46 47 #define readb(va) (*(volatile u_int8_t *) (va)) 48 #define readw(va) (*(volatile u_int16_t *) (va)) 49 #define readl(va) (*(volatile u_int32_t *) (va)) 50 51 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 52 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 53 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 54 55 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 56 57 static __inline void 58 breakpoint(void) 59 { 60 __asm __volatile("int $3"); 61 } 62 63 static __inline u_int 64 bsfl(u_int mask) 65 { 66 u_int result; 67 68 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 69 return (result); 70 } 71 72 static __inline u_int 73 bsrl(u_int mask) 74 { 75 u_int result; 76 77 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 78 return (result); 79 } 80 81 static __inline void 82 disable_intr(void) 83 { 84 __asm __volatile("cli" : : : "memory"); 85 } 86 87 static __inline void 88 do_cpuid(u_int ax, u_int *p) 89 { 90 __asm __volatile("cpuid" 91 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 92 : "0" (ax)); 93 } 94 95 static __inline void 96 cpuid_count(u_int ax, u_int cx, u_int *p) 97 { 98 __asm __volatile("cpuid" 99 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 100 : "0" (ax), "c" (cx)); 101 } 102 103 static __inline void 104 enable_intr(void) 105 { 106 __asm __volatile("sti"); 107 } 108 109 static inline void 110 cpu_monitor(const void *addr, int extensions, int hints) 111 { 112 __asm __volatile("monitor;" 113 : :"a" (addr), "c" (extensions), "d"(hints)); 114 } 115 116 static inline void 117 cpu_mwait(int extensions, int hints) 118 { 119 __asm __volatile("mwait;" : :"a" (hints), "c" (extensions)); 120 } 121 122 #ifdef _KERNEL 123 124 #define HAVE_INLINE_FFS 125 126 static __inline int 127 ffs(int mask) 128 { 129 /* 130 * Note that gcc-2's builtin ffs would be used if we didn't declare 131 * this inline or turn off the builtin. The builtin is faster but 132 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 133 * versions. 134 */ 135 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 136 } 137 138 #define HAVE_INLINE_FLS 139 140 static __inline int 141 fls(int mask) 142 { 143 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 144 } 145 146 #endif /* _KERNEL */ 147 148 static __inline void 149 halt(void) 150 { 151 __asm __volatile("hlt"); 152 } 153 154 #if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3 155 156 #define inb(port) inbv(port) 157 #define outb(port, data) outbv(port, data) 158 159 #else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */ 160 161 /* 162 * The following complications are to get around gcc not having a 163 * constraint letter for the range 0..255. We still put "d" in the 164 * constraint because "i" isn't a valid constraint when the port 165 * isn't constant. This only matters for -O0 because otherwise 166 * the non-working version gets optimized away. 167 * 168 * Use an expression-statement instead of a conditional expression 169 * because gcc-2.6.0 would promote the operands of the conditional 170 * and produce poor code for "if ((inb(var) & const1) == const2)". 171 * 172 * The unnecessary test `(port) < 0x10000' is to generate a warning if 173 * the `port' has type u_short or smaller. Such types are pessimal. 174 * This actually only works for signed types. The range check is 175 * careful to avoid generating warnings. 176 */ 177 #define inb(port) __extension__ ({ \ 178 u_char _data; \ 179 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 180 && (port) < 0x10000) \ 181 _data = inbc(port); \ 182 else \ 183 _data = inbv(port); \ 184 _data; }) 185 186 #define outb(port, data) ( \ 187 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 188 && (port) < 0x10000 \ 189 ? outbc(port, data) : outbv(port, data)) 190 191 static __inline u_char 192 inbc(u_int port) 193 { 194 u_char data; 195 196 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 197 return (data); 198 } 199 200 static __inline void 201 outbc(u_int port, u_char data) 202 { 203 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 204 } 205 206 #endif /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3*/ 207 208 static __inline u_char 209 inbv(u_int port) 210 { 211 u_char data; 212 /* 213 * We use %%dx and not %1 here because i/o is done at %dx and not at 214 * %edx, while gcc generates inferior code (movw instead of movl) 215 * if we tell it to load (u_short) port. 216 */ 217 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 218 return (data); 219 } 220 221 static __inline u_int 222 inl(u_int port) 223 { 224 u_int data; 225 226 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 227 return (data); 228 } 229 230 static __inline void 231 insb(u_int port, void *addr, size_t cnt) 232 { 233 __asm __volatile("cld; rep; insb" 234 : "+D" (addr), "+c" (cnt) 235 : "d" (port) 236 : "memory"); 237 } 238 239 static __inline void 240 insw(u_int port, void *addr, size_t cnt) 241 { 242 __asm __volatile("cld; rep; insw" 243 : "+D" (addr), "+c" (cnt) 244 : "d" (port) 245 : "memory"); 246 } 247 248 static __inline void 249 insl(u_int port, void *addr, size_t cnt) 250 { 251 __asm __volatile("cld; rep; insl" 252 : "+D" (addr), "+c" (cnt) 253 : "d" (port) 254 : "memory"); 255 } 256 257 static __inline void 258 invd(void) 259 { 260 __asm __volatile("invd"); 261 } 262 263 static __inline u_short 264 inw(u_int port) 265 { 266 u_short data; 267 268 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 269 return (data); 270 } 271 272 static __inline void 273 outbv(u_int port, u_char data) 274 { 275 u_char al; 276 /* 277 * Use an unnecessary assignment to help gcc's register allocator. 278 * This make a large difference for gcc-1.40 and a tiny difference 279 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 280 * best results. gcc-2.6.0 can't handle this. 281 */ 282 al = data; 283 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 284 } 285 286 static __inline void 287 outl(u_int port, u_int data) 288 { 289 /* 290 * outl() and outw() aren't used much so we haven't looked at 291 * possible micro-optimizations such as the unnecessary 292 * assignment for them. 293 */ 294 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 295 } 296 297 static __inline void 298 outsb(u_int port, const void *addr, size_t cnt) 299 { 300 __asm __volatile("cld; rep; outsb" 301 : "+S" (addr), "+c" (cnt) 302 : "d" (port)); 303 } 304 305 static __inline void 306 outsw(u_int port, const void *addr, size_t cnt) 307 { 308 __asm __volatile("cld; rep; outsw" 309 : "+S" (addr), "+c" (cnt) 310 : "d" (port)); 311 } 312 313 static __inline void 314 outsl(u_int port, const void *addr, size_t cnt) 315 { 316 __asm __volatile("cld; rep; outsl" 317 : "+S" (addr), "+c" (cnt) 318 : "d" (port)); 319 } 320 321 static __inline void 322 outw(u_int port, u_short data) 323 { 324 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 325 } 326 327 static __inline void 328 ia32_pause(void) 329 { 330 __asm __volatile("pause"); 331 } 332 333 static __inline u_int 334 read_eflags(void) 335 { 336 u_int ef; 337 338 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 339 return (ef); 340 } 341 342 static __inline uint64_t 343 rdmsr(u_int msr) 344 { 345 uint64_t rv; 346 347 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 348 return (rv); 349 } 350 351 static __inline uint64_t 352 rdpmc(u_int pmc) 353 { 354 uint64_t rv; 355 356 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 357 return (rv); 358 } 359 360 static __inline uint64_t 361 rdtsc(void) 362 { 363 uint64_t rv; 364 365 __asm __volatile("rdtsc" : "=A" (rv)); 366 return (rv); 367 } 368 369 static __inline void 370 wbinvd(void) 371 { 372 __asm __volatile("wbinvd"); 373 } 374 375 static __inline void 376 write_eflags(u_int ef) 377 { 378 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 379 } 380 381 static __inline void 382 wrmsr(u_int msr, uint64_t newval) 383 { 384 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 385 } 386 387 static __inline void 388 load_cr0(u_int data) 389 { 390 391 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 392 } 393 394 static __inline u_int 395 rcr0(void) 396 { 397 u_int data; 398 399 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 400 return (data); 401 } 402 403 static __inline u_int 404 rcr2(void) 405 { 406 u_int data; 407 408 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 409 return (data); 410 } 411 412 static __inline void 413 load_cr3(u_int data) 414 { 415 416 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 417 } 418 419 static __inline u_int 420 rcr3(void) 421 { 422 u_int data; 423 424 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 425 return (data); 426 } 427 428 static __inline void 429 load_cr4(u_int data) 430 { 431 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 432 } 433 434 static __inline u_int 435 rcr4(void) 436 { 437 u_int data; 438 439 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 440 return (data); 441 } 442 443 /* 444 * Global TLB flush (except for thise for pages marked PG_G) 445 */ 446 static __inline void 447 invltlb(void) 448 { 449 450 load_cr3(rcr3()); 451 } 452 453 /* 454 * TLB flush for an individual page (even if it has PG_G). 455 * Only works on 486+ CPUs (i386 does not have PG_G). 456 */ 457 static __inline void 458 invlpg(u_int addr) 459 { 460 461 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 462 } 463 464 static __inline u_int 465 rfs(void) 466 { 467 u_int sel; 468 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 469 return (sel); 470 } 471 472 static __inline uint64_t 473 rgdt(void) 474 { 475 uint64_t gdtr; 476 __asm __volatile("sgdt %0" : "=m" (gdtr)); 477 return (gdtr); 478 } 479 480 static __inline u_int 481 rgs(void) 482 { 483 u_int sel; 484 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 485 return (sel); 486 } 487 488 static __inline uint64_t 489 ridt(void) 490 { 491 uint64_t idtr; 492 __asm __volatile("sidt %0" : "=m" (idtr)); 493 return (idtr); 494 } 495 496 static __inline u_short 497 rldt(void) 498 { 499 u_short ldtr; 500 __asm __volatile("sldt %0" : "=g" (ldtr)); 501 return (ldtr); 502 } 503 504 static __inline u_int 505 rss(void) 506 { 507 u_int sel; 508 __asm __volatile("movl %%ss,%0" : "=rm" (sel)); 509 return (sel); 510 } 511 512 static __inline u_short 513 rtr(void) 514 { 515 u_short tr; 516 __asm __volatile("str %0" : "=g" (tr)); 517 return (tr); 518 } 519 520 static __inline void 521 load_fs(u_int sel) 522 { 523 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 524 } 525 526 static __inline void 527 load_gs(u_int sel) 528 { 529 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 530 } 531 532 static __inline void 533 lidt(struct region_descriptor *addr) 534 { 535 __asm __volatile("lidt (%0)" : : "r" (addr)); 536 } 537 538 static __inline void 539 lldt(u_short sel) 540 { 541 __asm __volatile("lldt %0" : : "r" (sel)); 542 } 543 544 static __inline void 545 ltr(u_short sel) 546 { 547 __asm __volatile("ltr %0" : : "r" (sel)); 548 } 549 550 static __inline u_int 551 rdr0(void) 552 { 553 u_int data; 554 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 555 return (data); 556 } 557 558 static __inline void 559 load_dr0(u_int dr0) 560 { 561 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 562 } 563 564 static __inline u_int 565 rdr1(void) 566 { 567 u_int data; 568 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 569 return (data); 570 } 571 572 static __inline void 573 load_dr1(u_int dr1) 574 { 575 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 576 } 577 578 static __inline u_int 579 rdr2(void) 580 { 581 u_int data; 582 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 583 return (data); 584 } 585 586 static __inline void 587 load_dr2(u_int dr2) 588 { 589 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 590 } 591 592 static __inline u_int 593 rdr3(void) 594 { 595 u_int data; 596 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 597 return (data); 598 } 599 600 static __inline void 601 load_dr3(u_int dr3) 602 { 603 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 604 } 605 606 static __inline u_int 607 rdr4(void) 608 { 609 u_int data; 610 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 611 return (data); 612 } 613 614 static __inline void 615 load_dr4(u_int dr4) 616 { 617 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 618 } 619 620 static __inline u_int 621 rdr5(void) 622 { 623 u_int data; 624 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 625 return (data); 626 } 627 628 static __inline void 629 load_dr5(u_int dr5) 630 { 631 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 632 } 633 634 static __inline u_int 635 rdr6(void) 636 { 637 u_int data; 638 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 639 return (data); 640 } 641 642 static __inline void 643 load_dr6(u_int dr6) 644 { 645 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 646 } 647 648 static __inline u_int 649 rdr7(void) 650 { 651 u_int data; 652 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 653 return (data); 654 } 655 656 static __inline void 657 load_dr7(u_int dr7) 658 { 659 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 660 } 661 662 static __inline register_t 663 intr_disable(void) 664 { 665 register_t eflags; 666 667 eflags = read_eflags(); 668 disable_intr(); 669 return (eflags); 670 } 671 672 static __inline void 673 intr_restore(register_t eflags) 674 { 675 write_eflags(eflags); 676 } 677 678 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 679 680 int breakpoint(void); 681 u_int bsfl(u_int mask); 682 u_int bsrl(u_int mask); 683 void disable_intr(void); 684 void do_cpuid(u_int ax, u_int *p); 685 void enable_intr(void); 686 void halt(void); 687 void ia32_pause(void); 688 u_char inb(u_int port); 689 u_int inl(u_int port); 690 void insb(u_int port, void *addr, size_t cnt); 691 void insl(u_int port, void *addr, size_t cnt); 692 void insw(u_int port, void *addr, size_t cnt); 693 register_t intr_disable(void); 694 void intr_restore(register_t ef); 695 void invd(void); 696 void invlpg(u_int addr); 697 void invltlb(void); 698 u_short inw(u_int port); 699 void lidt(struct region_descriptor *addr); 700 void lldt(u_short sel); 701 void load_cr0(u_int cr0); 702 void load_cr3(u_int cr3); 703 void load_cr4(u_int cr4); 704 void load_dr0(u_int dr0); 705 void load_dr1(u_int dr1); 706 void load_dr2(u_int dr2); 707 void load_dr3(u_int dr3); 708 void load_dr4(u_int dr4); 709 void load_dr5(u_int dr5); 710 void load_dr6(u_int dr6); 711 void load_dr7(u_int dr7); 712 void load_fs(u_int sel); 713 void load_gs(u_int sel); 714 void ltr(u_short sel); 715 void outb(u_int port, u_char data); 716 void outl(u_int port, u_int data); 717 void outsb(u_int port, const void *addr, size_t cnt); 718 void outsl(u_int port, const void *addr, size_t cnt); 719 void outsw(u_int port, const void *addr, size_t cnt); 720 void outw(u_int port, u_short data); 721 u_int rcr0(void); 722 u_int rcr2(void); 723 u_int rcr3(void); 724 u_int rcr4(void); 725 uint64_t rdmsr(u_int msr); 726 uint64_t rdpmc(u_int pmc); 727 u_int rdr0(void); 728 u_int rdr1(void); 729 u_int rdr2(void); 730 u_int rdr3(void); 731 u_int rdr4(void); 732 u_int rdr5(void); 733 u_int rdr6(void); 734 u_int rdr7(void); 735 uint64_t rdtsc(void); 736 u_int read_eflags(void); 737 u_int rfs(void); 738 uint64_t rgdt(void); 739 u_int rgs(void); 740 uint64_t ridt(void); 741 u_short rldt(void); 742 u_short rtr(void); 743 void wbinvd(void); 744 void write_eflags(u_int ef); 745 void wrmsr(u_int msr, uint64_t newval); 746 747 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 748 749 void reset_dbregs(void); 750 751 #endif /* !_MACHINE_CPUFUNC_H_ */ 752