1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * $FreeBSD$ 34 */ 35 36 /* 37 * Functions to provide access to special i386 instructions. 38 * This in included in sys/systm.h, and that file should be 39 * used in preference to this. 40 */ 41 42 #ifndef _MACHINE_CPUFUNC_H_ 43 #define _MACHINE_CPUFUNC_H_ 44 45 #include <sys/cdefs.h> 46 #include <machine/psl.h> 47 48 struct thread; 49 50 __BEGIN_DECLS 51 #define readb(va) (*(volatile u_int8_t *) (va)) 52 #define readw(va) (*(volatile u_int16_t *) (va)) 53 #define readl(va) (*(volatile u_int32_t *) (va)) 54 55 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 56 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 57 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 58 59 #ifdef __GNUC__ 60 61 static __inline void 62 breakpoint(void) 63 { 64 __asm __volatile("int $3"); 65 } 66 67 static __inline u_int 68 bsfl(u_int mask) 69 { 70 u_int result; 71 72 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 73 return (result); 74 } 75 76 static __inline u_int 77 bsrl(u_int mask) 78 { 79 u_int result; 80 81 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 82 return (result); 83 } 84 85 static __inline void 86 disable_intr(void) 87 { 88 __asm __volatile("cli" : : : "memory"); 89 } 90 91 static __inline void 92 do_cpuid(u_int ax, u_int *p) 93 { 94 __asm __volatile("cpuid" 95 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 96 : "0" (ax)); 97 } 98 99 static __inline void 100 enable_intr(void) 101 { 102 __asm __volatile("sti"); 103 } 104 105 #define HAVE_INLINE_FFS 106 107 static __inline int 108 ffs(int mask) 109 { 110 /* 111 * Note that gcc-2's builtin ffs would be used if we didn't declare 112 * this inline or turn off the builtin. The builtin is faster but 113 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 114 * versions. 115 */ 116 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 117 } 118 119 #define HAVE_INLINE_FLS 120 121 static __inline int 122 fls(int mask) 123 { 124 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 125 } 126 127 #if __GNUC__ < 2 128 129 #define inb(port) inbv(port) 130 #define outb(port, data) outbv(port, data) 131 132 #else /* __GNUC >= 2 */ 133 134 /* 135 * The following complications are to get around gcc not having a 136 * constraint letter for the range 0..255. We still put "d" in the 137 * constraint because "i" isn't a valid constraint when the port 138 * isn't constant. This only matters for -O0 because otherwise 139 * the non-working version gets optimized away. 140 * 141 * Use an expression-statement instead of a conditional expression 142 * because gcc-2.6.0 would promote the operands of the conditional 143 * and produce poor code for "if ((inb(var) & const1) == const2)". 144 * 145 * The unnecessary test `(port) < 0x10000' is to generate a warning if 146 * the `port' has type u_short or smaller. Such types are pessimal. 147 * This actually only works for signed types. The range check is 148 * careful to avoid generating warnings. 149 */ 150 #define inb(port) __extension__ ({ \ 151 u_char _data; \ 152 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 153 && (port) < 0x10000) \ 154 _data = inbc(port); \ 155 else \ 156 _data = inbv(port); \ 157 _data; }) 158 159 #define outb(port, data) ( \ 160 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 161 && (port) < 0x10000 \ 162 ? outbc(port, data) : outbv(port, data)) 163 164 static __inline u_char 165 inbc(u_int port) 166 { 167 u_char data; 168 169 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 170 return (data); 171 } 172 173 static __inline void 174 outbc(u_int port, u_char data) 175 { 176 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 177 } 178 179 #endif /* __GNUC <= 2 */ 180 181 static __inline u_char 182 inbv(u_int port) 183 { 184 u_char data; 185 /* 186 * We use %%dx and not %1 here because i/o is done at %dx and not at 187 * %edx, while gcc generates inferior code (movw instead of movl) 188 * if we tell it to load (u_short) port. 189 */ 190 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 191 return (data); 192 } 193 194 static __inline u_int 195 inl(u_int port) 196 { 197 u_int data; 198 199 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 200 return (data); 201 } 202 203 static __inline void 204 insb(u_int port, void *addr, size_t cnt) 205 { 206 __asm __volatile("cld; rep; insb" 207 : "+D" (addr), "+c" (cnt) 208 : "d" (port) 209 : "memory"); 210 } 211 212 static __inline void 213 insw(u_int port, void *addr, size_t cnt) 214 { 215 __asm __volatile("cld; rep; insw" 216 : "+D" (addr), "+c" (cnt) 217 : "d" (port) 218 : "memory"); 219 } 220 221 static __inline void 222 insl(u_int port, void *addr, size_t cnt) 223 { 224 __asm __volatile("cld; rep; insl" 225 : "+D" (addr), "+c" (cnt) 226 : "d" (port) 227 : "memory"); 228 } 229 230 static __inline void 231 invd(void) 232 { 233 __asm __volatile("invd"); 234 } 235 236 static __inline u_short 237 inw(u_int port) 238 { 239 u_short data; 240 241 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 242 return (data); 243 } 244 245 static __inline void 246 outbv(u_int port, u_char data) 247 { 248 u_char al; 249 /* 250 * Use an unnecessary assignment to help gcc's register allocator. 251 * This make a large difference for gcc-1.40 and a tiny difference 252 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 253 * best results. gcc-2.6.0 can't handle this. 254 */ 255 al = data; 256 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 257 } 258 259 static __inline void 260 outl(u_int port, u_int data) 261 { 262 /* 263 * outl() and outw() aren't used much so we haven't looked at 264 * possible micro-optimizations such as the unnecessary 265 * assignment for them. 266 */ 267 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 268 } 269 270 static __inline void 271 outsb(u_int port, const void *addr, size_t cnt) 272 { 273 __asm __volatile("cld; rep; outsb" 274 : "+S" (addr), "+c" (cnt) 275 : "d" (port)); 276 } 277 278 static __inline void 279 outsw(u_int port, const void *addr, size_t cnt) 280 { 281 __asm __volatile("cld; rep; outsw" 282 : "+S" (addr), "+c" (cnt) 283 : "d" (port)); 284 } 285 286 static __inline void 287 outsl(u_int port, const void *addr, size_t cnt) 288 { 289 __asm __volatile("cld; rep; outsl" 290 : "+S" (addr), "+c" (cnt) 291 : "d" (port)); 292 } 293 294 static __inline void 295 outw(u_int port, u_short data) 296 { 297 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 298 } 299 300 static __inline void 301 ia32_pause(void) 302 { 303 __asm __volatile("pause"); 304 } 305 306 static __inline u_int 307 read_eflags(void) 308 { 309 u_int ef; 310 311 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 312 return (ef); 313 } 314 315 static __inline u_int64_t 316 rdmsr(u_int msr) 317 { 318 u_int64_t rv; 319 320 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 321 return (rv); 322 } 323 324 static __inline u_int64_t 325 rdpmc(u_int pmc) 326 { 327 u_int64_t rv; 328 329 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 330 return (rv); 331 } 332 333 static __inline u_int64_t 334 rdtsc(void) 335 { 336 u_int64_t rv; 337 338 __asm __volatile("rdtsc" : "=A" (rv)); 339 return (rv); 340 } 341 342 static __inline void 343 wbinvd(void) 344 { 345 __asm __volatile("wbinvd"); 346 } 347 348 static __inline void 349 write_eflags(u_int ef) 350 { 351 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 352 } 353 354 static __inline void 355 wrmsr(u_int msr, u_int64_t newval) 356 { 357 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 358 } 359 360 static __inline void 361 load_cr0(u_int data) 362 { 363 364 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 365 } 366 367 static __inline u_int 368 rcr0(void) 369 { 370 u_int data; 371 372 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 373 return (data); 374 } 375 376 static __inline u_int 377 rcr2(void) 378 { 379 u_int data; 380 381 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 382 return (data); 383 } 384 385 static __inline void 386 load_cr3(u_int data) 387 { 388 389 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 390 } 391 392 static __inline u_int 393 rcr3(void) 394 { 395 u_int data; 396 397 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 398 return (data); 399 } 400 401 static __inline void 402 load_cr4(u_int data) 403 { 404 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 405 } 406 407 static __inline u_int 408 rcr4(void) 409 { 410 u_int data; 411 412 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 413 return (data); 414 } 415 416 /* 417 * Global TLB flush (except for thise for pages marked PG_G) 418 */ 419 static __inline void 420 invltlb(void) 421 { 422 423 load_cr3(rcr3()); 424 } 425 426 /* 427 * TLB flush for an individual page (even if it has PG_G). 428 * Only works on 486+ CPUs (i386 does not have PG_G). 429 */ 430 static __inline void 431 invlpg(u_int addr) 432 { 433 434 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 435 } 436 437 static __inline u_int 438 rfs(void) 439 { 440 u_int sel; 441 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 442 return (sel); 443 } 444 445 static __inline u_int 446 rgs(void) 447 { 448 u_int sel; 449 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 450 return (sel); 451 } 452 453 static __inline void 454 load_fs(u_int sel) 455 { 456 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 457 } 458 459 static __inline void 460 load_gs(u_int sel) 461 { 462 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 463 } 464 465 static __inline u_int 466 rdr0(void) 467 { 468 u_int data; 469 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 470 return (data); 471 } 472 473 static __inline void 474 load_dr0(u_int dr0) 475 { 476 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 477 } 478 479 static __inline u_int 480 rdr1(void) 481 { 482 u_int data; 483 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 484 return (data); 485 } 486 487 static __inline void 488 load_dr1(u_int dr1) 489 { 490 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 491 } 492 493 static __inline u_int 494 rdr2(void) 495 { 496 u_int data; 497 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 498 return (data); 499 } 500 501 static __inline void 502 load_dr2(u_int dr2) 503 { 504 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 505 } 506 507 static __inline u_int 508 rdr3(void) 509 { 510 u_int data; 511 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 512 return (data); 513 } 514 515 static __inline void 516 load_dr3(u_int dr3) 517 { 518 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 519 } 520 521 static __inline u_int 522 rdr4(void) 523 { 524 u_int data; 525 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 526 return (data); 527 } 528 529 static __inline void 530 load_dr4(u_int dr4) 531 { 532 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 533 } 534 535 static __inline u_int 536 rdr5(void) 537 { 538 u_int data; 539 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 540 return (data); 541 } 542 543 static __inline void 544 load_dr5(u_int dr5) 545 { 546 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 547 } 548 549 static __inline u_int 550 rdr6(void) 551 { 552 u_int data; 553 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 554 return (data); 555 } 556 557 static __inline void 558 load_dr6(u_int dr6) 559 { 560 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 561 } 562 563 static __inline u_int 564 rdr7(void) 565 { 566 u_int data; 567 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 568 return (data); 569 } 570 571 static __inline void 572 load_dr7(u_int dr7) 573 { 574 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 575 } 576 577 static __inline register_t 578 intr_disable(void) 579 { 580 register_t eflags; 581 582 eflags = read_eflags(); 583 disable_intr(); 584 return (eflags); 585 } 586 587 static __inline void 588 intr_restore(register_t eflags) 589 { 590 write_eflags(eflags); 591 } 592 593 #else /* !__GNUC__ */ 594 595 int breakpoint(void); 596 u_int bsfl(u_int mask); 597 u_int bsrl(u_int mask); 598 void cpu_invlpg(u_int addr); 599 void cpu_invlpg_range(u_int start, u_int end); 600 void disable_intr(void); 601 void do_cpuid(u_int ax, u_int *p); 602 void enable_intr(void); 603 u_char inb(u_int port); 604 u_int inl(u_int port); 605 void insb(u_int port, void *addr, size_t cnt); 606 void insl(u_int port, void *addr, size_t cnt); 607 void insw(u_int port, void *addr, size_t cnt); 608 void invd(void); 609 void invlpg(u_int addr); 610 void invlpg_range(u_int start, u_int end); 611 void invltlb(void); 612 u_short inw(u_int port); 613 void load_cr0(u_int cr0); 614 void load_cr3(u_int cr3); 615 void load_cr4(u_int cr4); 616 void load_fs(u_int sel); 617 void load_gs(u_int sel); 618 void outb(u_int port, u_char data); 619 void outl(u_int port, u_int data); 620 void outsb(u_int port, void *addr, size_t cnt); 621 void outsl(u_int port, void *addr, size_t cnt); 622 void outsw(u_int port, void *addr, size_t cnt); 623 void outw(u_int port, u_short data); 624 void ia32_pause(void); 625 u_int rcr0(void); 626 u_int rcr2(void); 627 u_int rcr3(void); 628 u_int rcr4(void); 629 u_int rfs(void); 630 u_int rgs(void); 631 u_int64_t rdmsr(u_int msr); 632 u_int64_t rdpmc(u_int pmc); 633 u_int64_t rdtsc(void); 634 u_int read_eflags(void); 635 void wbinvd(void); 636 void write_eflags(u_int ef); 637 void wrmsr(u_int msr, u_int64_t newval); 638 u_int rdr0(void); 639 void load_dr0(u_int dr0); 640 u_int rdr1(void); 641 void load_dr1(u_int dr1); 642 u_int rdr2(void); 643 void load_dr2(u_int dr2); 644 u_int rdr3(void); 645 void load_dr3(u_int dr3); 646 u_int rdr4(void); 647 void load_dr4(u_int dr4); 648 u_int rdr5(void); 649 void load_dr5(u_int dr5); 650 u_int rdr6(void); 651 void load_dr6(u_int dr6); 652 u_int rdr7(void); 653 void load_dr7(u_int dr7); 654 register_t intr_disable(void); 655 void intr_restore(register_t ef); 656 657 #endif /* __GNUC__ */ 658 659 void ltr(u_short sel); 660 void reset_dbregs(void); 661 662 __END_DECLS 663 664 #endif /* !_MACHINE_CPUFUNC_H_ */ 665