1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 /* 35 * Functions to provide access to special i386 instructions. 36 * This in included in sys/systm.h, and that file should be 37 * used in preference to this. 38 */ 39 40 #ifndef _MACHINE_CPUFUNC_H_ 41 #define _MACHINE_CPUFUNC_H_ 42 43 struct region_descriptor; 44 45 #define readb(va) (*(volatile uint8_t *) (va)) 46 #define readw(va) (*(volatile uint16_t *) (va)) 47 #define readl(va) (*(volatile uint32_t *) (va)) 48 49 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 50 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 51 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 52 53 static __inline void 54 breakpoint(void) 55 { 56 __asm __volatile("int $3"); 57 } 58 59 static __inline __pure2 u_int 60 bsfl(u_int mask) 61 { 62 u_int result; 63 64 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 65 return (result); 66 } 67 68 static __inline __pure2 u_int 69 bsrl(u_int mask) 70 { 71 u_int result; 72 73 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 74 return (result); 75 } 76 77 static __inline void 78 clflush(u_long addr) 79 { 80 81 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 82 } 83 84 static __inline void 85 clflushopt(u_long addr) 86 { 87 88 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 89 } 90 91 static __inline void 92 clts(void) 93 { 94 95 __asm __volatile("clts"); 96 } 97 98 static __inline void 99 disable_intr(void) 100 { 101 __asm __volatile("cli" : : : "memory"); 102 } 103 104 #ifdef _KERNEL 105 static __inline void 106 do_cpuid(u_int ax, u_int *p) 107 { 108 __asm __volatile("cpuid" 109 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 110 : "0" (ax)); 111 } 112 113 static __inline void 114 cpuid_count(u_int ax, u_int cx, u_int *p) 115 { 116 __asm __volatile("cpuid" 117 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 118 : "0" (ax), "c" (cx)); 119 } 120 #else 121 static __inline void 122 do_cpuid(u_int ax, u_int *p) 123 { 124 __asm __volatile( 125 "pushl\t%%ebx\n\t" 126 "cpuid\n\t" 127 "movl\t%%ebx,%1\n\t" 128 "popl\t%%ebx" 129 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 130 : "0" (ax)); 131 } 132 133 static __inline void 134 cpuid_count(u_int ax, u_int cx, u_int *p) 135 { 136 __asm __volatile( 137 "pushl\t%%ebx\n\t" 138 "cpuid\n\t" 139 "movl\t%%ebx,%1\n\t" 140 "popl\t%%ebx" 141 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 142 : "0" (ax), "c" (cx)); 143 } 144 #endif 145 146 static __inline void 147 enable_intr(void) 148 { 149 __asm __volatile("sti"); 150 } 151 152 static __inline void 153 cpu_monitor(const void *addr, u_long extensions, u_int hints) 154 { 155 __asm __volatile("monitor" 156 : : "a" (addr), "c" (extensions), "d" (hints)); 157 } 158 159 static __inline void 160 cpu_mwait(u_long extensions, u_int hints) 161 { 162 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 163 } 164 165 static __inline void 166 lfence(void) 167 { 168 __asm __volatile("lfence" : : : "memory"); 169 } 170 171 static __inline void 172 mfence(void) 173 { 174 __asm __volatile("mfence" : : : "memory"); 175 } 176 177 static __inline void 178 sfence(void) 179 { 180 __asm __volatile("sfence" : : : "memory"); 181 } 182 183 static __inline void 184 halt(void) 185 { 186 __asm __volatile("hlt"); 187 } 188 189 static __inline u_char 190 inb(u_int port) 191 { 192 u_char data; 193 194 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 195 return (data); 196 } 197 198 static __inline u_int 199 inl(u_int port) 200 { 201 u_int data; 202 203 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 204 return (data); 205 } 206 207 static __inline void 208 insb(u_int port, void *addr, size_t count) 209 { 210 __asm __volatile("cld; rep; insb" 211 : "+D" (addr), "+c" (count) 212 : "d" (port) 213 : "memory"); 214 } 215 216 static __inline void 217 insw(u_int port, void *addr, size_t count) 218 { 219 __asm __volatile("cld; rep; insw" 220 : "+D" (addr), "+c" (count) 221 : "d" (port) 222 : "memory"); 223 } 224 225 static __inline void 226 insl(u_int port, void *addr, size_t count) 227 { 228 __asm __volatile("cld; rep; insl" 229 : "+D" (addr), "+c" (count) 230 : "d" (port) 231 : "memory"); 232 } 233 234 static __inline void 235 invd(void) 236 { 237 __asm __volatile("invd"); 238 } 239 240 static __inline u_short 241 inw(u_int port) 242 { 243 u_short data; 244 245 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 246 return (data); 247 } 248 249 static __inline void 250 outb(u_int port, u_char data) 251 { 252 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 253 } 254 255 static __inline void 256 outl(u_int port, u_int data) 257 { 258 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 259 } 260 261 static __inline void 262 outsb(u_int port, const void *addr, size_t count) 263 { 264 __asm __volatile("cld; rep; outsb" 265 : "+S" (addr), "+c" (count) 266 : "d" (port)); 267 } 268 269 static __inline void 270 outsw(u_int port, const void *addr, size_t count) 271 { 272 __asm __volatile("cld; rep; outsw" 273 : "+S" (addr), "+c" (count) 274 : "d" (port)); 275 } 276 277 static __inline void 278 outsl(u_int port, const void *addr, size_t count) 279 { 280 __asm __volatile("cld; rep; outsl" 281 : "+S" (addr), "+c" (count) 282 : "d" (port)); 283 } 284 285 static __inline void 286 outw(u_int port, u_short data) 287 { 288 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 289 } 290 291 static __inline void 292 ia32_pause(void) 293 { 294 __asm __volatile("pause"); 295 } 296 297 static __inline u_int 298 read_eflags(void) 299 { 300 u_int ef; 301 302 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 303 return (ef); 304 } 305 306 static __inline uint64_t 307 rdmsr(u_int msr) 308 { 309 uint64_t rv; 310 311 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 312 return (rv); 313 } 314 315 static __inline uint32_t 316 rdmsr32(u_int msr) 317 { 318 uint32_t low; 319 320 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 321 return (low); 322 } 323 324 static __inline uint64_t 325 rdpmc(u_int pmc) 326 { 327 uint64_t rv; 328 329 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 330 return (rv); 331 } 332 333 static __inline uint64_t 334 rdtsc(void) 335 { 336 uint64_t rv; 337 338 __asm __volatile("rdtsc" : "=A" (rv)); 339 return (rv); 340 } 341 342 static __inline uint64_t 343 rdtsc_ordered_lfence(void) 344 { 345 lfence(); 346 return (rdtsc()); 347 } 348 349 static __inline uint64_t 350 rdtsc_ordered_mfence(void) 351 { 352 mfence(); 353 return (rdtsc()); 354 } 355 356 static __inline uint64_t 357 rdtscp(void) 358 { 359 uint64_t rv; 360 361 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx"); 362 return (rv); 363 } 364 365 static __inline uint64_t 366 rdtscp_aux(uint32_t *aux) 367 { 368 uint64_t rv; 369 370 __asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux)); 371 return (rv); 372 } 373 374 static __inline uint32_t 375 rdtsc32(void) 376 { 377 uint32_t rv; 378 379 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 380 return (rv); 381 } 382 383 static __inline uint32_t 384 rdtscp32(void) 385 { 386 uint32_t rv; 387 388 __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx"); 389 return (rv); 390 } 391 392 static __inline void 393 wbinvd(void) 394 { 395 __asm __volatile("wbinvd"); 396 } 397 398 static __inline void 399 write_eflags(u_int ef) 400 { 401 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 402 } 403 404 static __inline void 405 wrmsr(u_int msr, uint64_t newval) 406 { 407 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 408 } 409 410 static __inline void 411 load_cr0(u_int data) 412 { 413 414 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 415 } 416 417 static __inline u_int 418 rcr0(void) 419 { 420 u_int data; 421 422 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 423 return (data); 424 } 425 426 static __inline u_int 427 rcr2(void) 428 { 429 u_int data; 430 431 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 432 return (data); 433 } 434 435 static __inline void 436 load_cr3(u_int data) 437 { 438 439 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 440 } 441 442 static __inline u_int 443 rcr3(void) 444 { 445 u_int data; 446 447 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 448 return (data); 449 } 450 451 static __inline void 452 load_cr4(u_int data) 453 { 454 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 455 } 456 457 static __inline u_int 458 rcr4(void) 459 { 460 u_int data; 461 462 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 463 return (data); 464 } 465 466 static __inline uint64_t 467 rxcr(u_int reg) 468 { 469 u_int low, high; 470 471 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 472 return (low | ((uint64_t)high << 32)); 473 } 474 475 static __inline void 476 load_xcr(u_int reg, uint64_t val) 477 { 478 u_int low, high; 479 480 low = val; 481 high = val >> 32; 482 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 483 } 484 485 /* 486 * Global TLB flush (except for thise for pages marked PG_G) 487 */ 488 static __inline void 489 invltlb(void) 490 { 491 492 load_cr3(rcr3()); 493 } 494 495 /* 496 * TLB flush for an individual page (even if it has PG_G). 497 * Only works on 486+ CPUs (i386 does not have PG_G). 498 */ 499 static __inline void 500 invlpg(u_int addr) 501 { 502 503 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 504 } 505 506 static __inline u_short 507 rfs(void) 508 { 509 u_short sel; 510 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 511 return (sel); 512 } 513 514 static __inline uint64_t 515 rgdt(void) 516 { 517 uint64_t gdtr; 518 __asm __volatile("sgdt %0" : "=m" (gdtr)); 519 return (gdtr); 520 } 521 522 static __inline u_short 523 rgs(void) 524 { 525 u_short sel; 526 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 527 return (sel); 528 } 529 530 static __inline uint64_t 531 ridt(void) 532 { 533 uint64_t idtr; 534 __asm __volatile("sidt %0" : "=m" (idtr)); 535 return (idtr); 536 } 537 538 static __inline u_short 539 rldt(void) 540 { 541 u_short ldtr; 542 __asm __volatile("sldt %0" : "=g" (ldtr)); 543 return (ldtr); 544 } 545 546 static __inline u_short 547 rss(void) 548 { 549 u_short sel; 550 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 551 return (sel); 552 } 553 554 static __inline u_short 555 rtr(void) 556 { 557 u_short tr; 558 __asm __volatile("str %0" : "=g" (tr)); 559 return (tr); 560 } 561 562 static __inline void 563 load_fs(u_short sel) 564 { 565 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 566 } 567 568 static __inline void 569 load_gs(u_short sel) 570 { 571 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 572 } 573 574 static __inline void 575 lidt(struct region_descriptor *addr) 576 { 577 __asm __volatile("lidt (%0)" : : "r" (addr)); 578 } 579 580 static __inline void 581 lldt(u_short sel) 582 { 583 __asm __volatile("lldt %0" : : "r" (sel)); 584 } 585 586 static __inline void 587 ltr(u_short sel) 588 { 589 __asm __volatile("ltr %0" : : "r" (sel)); 590 } 591 592 static __inline u_int 593 rdr0(void) 594 { 595 u_int data; 596 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 597 return (data); 598 } 599 600 static __inline void 601 load_dr0(u_int dr0) 602 { 603 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 604 } 605 606 static __inline u_int 607 rdr1(void) 608 { 609 u_int data; 610 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 611 return (data); 612 } 613 614 static __inline void 615 load_dr1(u_int dr1) 616 { 617 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 618 } 619 620 static __inline u_int 621 rdr2(void) 622 { 623 u_int data; 624 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 625 return (data); 626 } 627 628 static __inline void 629 load_dr2(u_int dr2) 630 { 631 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 632 } 633 634 static __inline u_int 635 rdr3(void) 636 { 637 u_int data; 638 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 639 return (data); 640 } 641 642 static __inline void 643 load_dr3(u_int dr3) 644 { 645 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 646 } 647 648 static __inline u_int 649 rdr6(void) 650 { 651 u_int data; 652 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 653 return (data); 654 } 655 656 static __inline void 657 load_dr6(u_int dr6) 658 { 659 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 660 } 661 662 static __inline u_int 663 rdr7(void) 664 { 665 u_int data; 666 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 667 return (data); 668 } 669 670 static __inline void 671 load_dr7(u_int dr7) 672 { 673 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 674 } 675 676 static __inline u_char 677 read_cyrix_reg(u_char reg) 678 { 679 outb(0x22, reg); 680 return inb(0x23); 681 } 682 683 static __inline void 684 write_cyrix_reg(u_char reg, u_char data) 685 { 686 outb(0x22, reg); 687 outb(0x23, data); 688 } 689 690 static __inline register_t 691 intr_disable(void) 692 { 693 register_t eflags; 694 695 eflags = read_eflags(); 696 disable_intr(); 697 return (eflags); 698 } 699 700 static __inline void 701 intr_restore(register_t eflags) 702 { 703 write_eflags(eflags); 704 } 705 706 static __inline uint32_t 707 rdpkru(void) 708 { 709 uint32_t res; 710 711 __asm __volatile("rdpkru" : "=a" (res) : "c" (0) : "edx"); 712 return (res); 713 } 714 715 static __inline void 716 wrpkru(uint32_t mask) 717 { 718 719 __asm __volatile("wrpkru" : : "a" (mask), "c" (0), "d" (0)); 720 } 721 722 void reset_dbregs(void); 723 724 #ifdef _KERNEL 725 int rdmsr_safe(u_int msr, uint64_t *val); 726 int wrmsr_safe(u_int msr, uint64_t newval); 727 #endif 728 729 #endif /* !_MACHINE_CPUFUNC_H_ */ 730