1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * $FreeBSD$ 34 */ 35 36 /* 37 * Functions to provide access to special i386 instructions. 38 * This in included in sys/systm.h, and that file should be 39 * used in preference to this. 40 */ 41 42 #ifndef _MACHINE_CPUFUNC_H_ 43 #define _MACHINE_CPUFUNC_H_ 44 45 #include <sys/cdefs.h> 46 #include <machine/psl.h> 47 48 struct thread; 49 struct region_descriptor; 50 51 __BEGIN_DECLS 52 #define readb(va) (*(volatile u_int8_t *) (va)) 53 #define readw(va) (*(volatile u_int16_t *) (va)) 54 #define readl(va) (*(volatile u_int32_t *) (va)) 55 #define readq(va) (*(volatile u_int64_t *) (va)) 56 57 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 58 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 59 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 60 #define writeq(va, d) (*(volatile u_int64_t *) (va) = (d)) 61 62 #ifdef __GNUC__ 63 64 static __inline void 65 breakpoint(void) 66 { 67 __asm __volatile("int $3"); 68 } 69 70 static __inline u_int 71 bsfl(u_int mask) 72 { 73 u_int result; 74 75 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 76 return (result); 77 } 78 79 static __inline u_int 80 bsrl(u_int mask) 81 { 82 u_int result; 83 84 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 85 return (result); 86 } 87 88 static __inline void 89 disable_intr(void) 90 { 91 __asm __volatile("cli" : : : "memory"); 92 } 93 94 static __inline void 95 do_cpuid(u_int ax, u_int *p) 96 { 97 __asm __volatile("cpuid" 98 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 99 : "0" (ax)); 100 } 101 102 static __inline void 103 enable_intr(void) 104 { 105 __asm __volatile("sti"); 106 } 107 108 #define HAVE_INLINE_FFS 109 110 static __inline int 111 ffs(int mask) 112 { 113 /* 114 * Note that gcc-2's builtin ffs would be used if we didn't declare 115 * this inline or turn off the builtin. The builtin is faster but 116 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 117 * versions. 118 */ 119 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 120 } 121 122 #define HAVE_INLINE_FLS 123 124 static __inline int 125 fls(int mask) 126 { 127 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 128 } 129 130 static __inline void 131 halt(void) 132 { 133 __asm __volatile("hlt"); 134 } 135 136 #if __GNUC__ < 2 137 138 #define inb(port) inbv(port) 139 #define outb(port, data) outbv(port, data) 140 141 #else /* __GNUC >= 2 */ 142 143 /* 144 * The following complications are to get around gcc not having a 145 * constraint letter for the range 0..255. We still put "d" in the 146 * constraint because "i" isn't a valid constraint when the port 147 * isn't constant. This only matters for -O0 because otherwise 148 * the non-working version gets optimized away. 149 * 150 * Use an expression-statement instead of a conditional expression 151 * because gcc-2.6.0 would promote the operands of the conditional 152 * and produce poor code for "if ((inb(var) & const1) == const2)". 153 * 154 * The unnecessary test `(port) < 0x10000' is to generate a warning if 155 * the `port' has type u_short or smaller. Such types are pessimal. 156 * This actually only works for signed types. The range check is 157 * careful to avoid generating warnings. 158 */ 159 #define inb(port) __extension__ ({ \ 160 u_char _data; \ 161 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 162 && (port) < 0x10000) \ 163 _data = inbc(port); \ 164 else \ 165 _data = inbv(port); \ 166 _data; }) 167 168 #define outb(port, data) ( \ 169 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 170 && (port) < 0x10000 \ 171 ? outbc(port, data) : outbv(port, data)) 172 173 static __inline u_char 174 inbc(u_int port) 175 { 176 u_char data; 177 178 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 179 return (data); 180 } 181 182 static __inline void 183 outbc(u_int port, u_char data) 184 { 185 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 186 } 187 188 #endif /* __GNUC <= 2 */ 189 190 static __inline u_char 191 inbv(u_int port) 192 { 193 u_char data; 194 /* 195 * We use %%dx and not %1 here because i/o is done at %dx and not at 196 * %edx, while gcc generates inferior code (movw instead of movl) 197 * if we tell it to load (u_short) port. 198 */ 199 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 200 return (data); 201 } 202 203 static __inline u_int 204 inl(u_int port) 205 { 206 u_int data; 207 208 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 209 return (data); 210 } 211 212 static __inline void 213 insb(u_int port, void *addr, size_t cnt) 214 { 215 __asm __volatile("cld; rep; insb" 216 : "+D" (addr), "+c" (cnt) 217 : "d" (port) 218 : "memory"); 219 } 220 221 static __inline void 222 insw(u_int port, void *addr, size_t cnt) 223 { 224 __asm __volatile("cld; rep; insw" 225 : "+D" (addr), "+c" (cnt) 226 : "d" (port) 227 : "memory"); 228 } 229 230 static __inline void 231 insl(u_int port, void *addr, size_t cnt) 232 { 233 __asm __volatile("cld; rep; insl" 234 : "+D" (addr), "+c" (cnt) 235 : "d" (port) 236 : "memory"); 237 } 238 239 static __inline void 240 invd(void) 241 { 242 __asm __volatile("invd"); 243 } 244 245 static __inline u_short 246 inw(u_int port) 247 { 248 u_short data; 249 250 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 251 return (data); 252 } 253 254 static __inline void 255 outbv(u_int port, u_char data) 256 { 257 u_char al; 258 /* 259 * Use an unnecessary assignment to help gcc's register allocator. 260 * This make a large difference for gcc-1.40 and a tiny difference 261 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 262 * best results. gcc-2.6.0 can't handle this. 263 */ 264 al = data; 265 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 266 } 267 268 static __inline void 269 outl(u_int port, u_int data) 270 { 271 /* 272 * outl() and outw() aren't used much so we haven't looked at 273 * possible micro-optimizations such as the unnecessary 274 * assignment for them. 275 */ 276 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 277 } 278 279 static __inline void 280 outsb(u_int port, const void *addr, size_t cnt) 281 { 282 __asm __volatile("cld; rep; outsb" 283 : "+S" (addr), "+c" (cnt) 284 : "d" (port)); 285 } 286 287 static __inline void 288 outsw(u_int port, const void *addr, size_t cnt) 289 { 290 __asm __volatile("cld; rep; outsw" 291 : "+S" (addr), "+c" (cnt) 292 : "d" (port)); 293 } 294 295 static __inline void 296 outsl(u_int port, const void *addr, size_t cnt) 297 { 298 __asm __volatile("cld; rep; outsl" 299 : "+S" (addr), "+c" (cnt) 300 : "d" (port)); 301 } 302 303 static __inline void 304 outw(u_int port, u_short data) 305 { 306 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 307 } 308 309 static __inline void 310 ia32_pause(void) 311 { 312 __asm __volatile("pause"); 313 } 314 315 static __inline u_long 316 read_rflags(void) 317 { 318 u_long rf; 319 320 __asm __volatile("pushfq; popq %0" : "=r" (rf)); 321 return (rf); 322 } 323 324 static __inline u_int64_t 325 rdmsr(u_int msr) 326 { 327 u_int32_t low, high; 328 329 __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr)); 330 return (low | ((u_int64_t)high << 32)); 331 } 332 333 static __inline u_int64_t 334 rdpmc(u_int pmc) 335 { 336 u_int32_t low, high; 337 338 __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc)); 339 return (low | ((u_int64_t)high << 32)); 340 } 341 342 static __inline u_int64_t 343 rdtsc(void) 344 { 345 u_int32_t low, high; 346 347 __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 348 return (low | ((u_int64_t)high << 32)); 349 } 350 351 static __inline void 352 wbinvd(void) 353 { 354 __asm __volatile("wbinvd"); 355 } 356 357 static __inline void 358 write_rflags(u_long rf) 359 { 360 __asm __volatile("pushq %0; popfq" : : "r" (rf)); 361 } 362 363 static __inline void 364 wrmsr(u_int msr, u_int64_t newval) 365 { 366 u_int32_t low, high; 367 368 low = newval; 369 high = newval >> 32; 370 __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr)); 371 } 372 373 static __inline void 374 load_cr0(u_long data) 375 { 376 377 __asm __volatile("movq %0,%%cr0" : : "r" (data)); 378 } 379 380 static __inline u_long 381 rcr0(void) 382 { 383 u_long data; 384 385 __asm __volatile("movq %%cr0,%0" : "=r" (data)); 386 return (data); 387 } 388 389 static __inline u_long 390 rcr2(void) 391 { 392 u_long data; 393 394 __asm __volatile("movq %%cr2,%0" : "=r" (data)); 395 return (data); 396 } 397 398 static __inline void 399 load_cr3(u_long data) 400 { 401 402 __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory"); 403 } 404 405 static __inline u_long 406 rcr3(void) 407 { 408 u_long data; 409 410 __asm __volatile("movq %%cr3,%0" : "=r" (data)); 411 return (data); 412 } 413 414 static __inline void 415 load_cr4(u_long data) 416 { 417 __asm __volatile("movq %0,%%cr4" : : "r" (data)); 418 } 419 420 static __inline u_long 421 rcr4(void) 422 { 423 u_long data; 424 425 __asm __volatile("movq %%cr4,%0" : "=r" (data)); 426 return (data); 427 } 428 429 /* 430 * Global TLB flush (except for thise for pages marked PG_G) 431 */ 432 static __inline void 433 invltlb(void) 434 { 435 436 load_cr3(rcr3()); 437 } 438 439 /* 440 * TLB flush for an individual page (even if it has PG_G). 441 * Only works on 486+ CPUs (i386 does not have PG_G). 442 */ 443 static __inline void 444 invlpg(u_long addr) 445 { 446 447 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 448 } 449 450 /* XXX these are replaced with rdmsr/wrmsr */ 451 static __inline u_int 452 rfs(void) 453 { 454 u_int sel; 455 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 456 return (sel); 457 } 458 459 static __inline u_int 460 rgs(void) 461 { 462 u_int sel; 463 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 464 return (sel); 465 } 466 467 static __inline void 468 load_fs(u_int sel) 469 { 470 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 471 } 472 473 static __inline void 474 load_gs(u_int sel) 475 { 476 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 477 } 478 479 /* void lidt(struct region_descriptor *addr); */ 480 static __inline void 481 lidt(struct region_descriptor *addr) 482 { 483 __asm __volatile("lidt (%0)" : : "r" (addr)); 484 } 485 486 /* void lldt(u_short sel); */ 487 static __inline void 488 lldt(u_short sel) 489 { 490 __asm __volatile("lldt %0" : : "r" (sel)); 491 } 492 493 /* void ltr(u_short sel); */ 494 static __inline void 495 ltr(u_short sel) 496 { 497 __asm __volatile("ltr %0" : : "r" (sel)); 498 } 499 500 static __inline register_t 501 intr_disable(void) 502 { 503 register_t rflags; 504 505 rflags = read_rflags(); 506 disable_intr(); 507 return (rflags); 508 } 509 510 static __inline void 511 intr_restore(register_t rflags) 512 { 513 write_rflags(rflags); 514 } 515 516 #else /* !__GNUC__ */ 517 518 int breakpoint(void); 519 u_int bsfl(u_int mask); 520 u_int bsrl(u_int mask); 521 void cpu_invlpg(u_long addr); 522 void cpu_invlpg_range(u_long start, u_long end); 523 void disable_intr(void); 524 void do_cpuid(u_int ax, u_int *p); 525 void enable_intr(void); 526 void halt(void); 527 u_char inb(u_int port); 528 u_int inl(u_int port); 529 void insb(u_int port, void *addr, size_t cnt); 530 void insl(u_int port, void *addr, size_t cnt); 531 void insw(u_int port, void *addr, size_t cnt); 532 void invd(void); 533 void invlpg(u_int addr); 534 void invlpg_range(u_int start, u_int end); 535 void invltlb(void); 536 u_short inw(u_int port); 537 void load_cr0(u_int cr0); 538 void load_cr3(u_int cr3); 539 void load_cr4(u_int cr4); 540 void load_fs(u_int sel); 541 void load_gs(u_int sel); 542 struct region_descriptor; 543 void lidt(struct region_descriptor *addr); 544 void lldt(u_short sel); 545 void ltr(u_short sel); 546 void outb(u_int port, u_char data); 547 void outl(u_int port, u_int data); 548 void outsb(u_int port, void *addr, size_t cnt); 549 void outsl(u_int port, void *addr, size_t cnt); 550 void outsw(u_int port, void *addr, size_t cnt); 551 void outw(u_int port, u_short data); 552 void ia32_pause(void); 553 u_int rcr0(void); 554 u_int rcr2(void); 555 u_int rcr3(void); 556 u_int rcr4(void); 557 u_int rfs(void); 558 u_int rgs(void); 559 u_int64_t rdmsr(u_int msr); 560 u_int64_t rdpmc(u_int pmc); 561 u_int64_t rdtsc(void); 562 u_int read_rflags(void); 563 void wbinvd(void); 564 void write_rflags(u_int rf); 565 void wrmsr(u_int msr, u_int64_t newval); 566 void load_dr7(u_int dr7); 567 register_t intr_disable(void); 568 void intr_restore(register_t rf); 569 570 #endif /* __GNUC__ */ 571 572 void reset_dbregs(void); 573 574 __END_DECLS 575 576 #endif /* !_MACHINE_CPUFUNC_H_ */ 577