1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 struct region_descriptor; 42 43 #define readb(va) (*(volatile uint8_t *) (va)) 44 #define readw(va) (*(volatile uint16_t *) (va)) 45 #define readl(va) (*(volatile uint32_t *) (va)) 46 47 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 48 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 49 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 50 51 static __inline void 52 breakpoint(void) 53 { 54 __asm __volatile("int $3"); 55 } 56 57 static __inline __pure2 u_int 58 bsfl(u_int mask) 59 { 60 u_int result; 61 62 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 63 return (result); 64 } 65 66 static __inline void 67 clflush(u_long addr) 68 { 69 70 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 71 } 72 73 static __inline void 74 clflushopt(u_long addr) 75 { 76 77 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 78 } 79 80 static __inline void 81 clts(void) 82 { 83 84 __asm __volatile("clts"); 85 } 86 87 static __inline void 88 disable_intr(void) 89 { 90 __asm __volatile("cli" : : : "memory"); 91 } 92 93 #ifdef _KERNEL 94 static __inline void 95 do_cpuid(u_int ax, u_int *p) 96 { 97 __asm __volatile("cpuid" 98 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 99 : "0" (ax)); 100 } 101 102 static __inline void 103 cpuid_count(u_int ax, u_int cx, u_int *p) 104 { 105 __asm __volatile("cpuid" 106 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 107 : "0" (ax), "c" (cx)); 108 } 109 #else 110 static __inline void 111 do_cpuid(u_int ax, u_int *p) 112 { 113 __asm __volatile( 114 "pushl\t%%ebx\n\t" 115 "cpuid\n\t" 116 "movl\t%%ebx,%1\n\t" 117 "popl\t%%ebx" 118 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 119 : "0" (ax)); 120 } 121 122 static __inline void 123 cpuid_count(u_int ax, u_int cx, u_int *p) 124 { 125 __asm __volatile( 126 "pushl\t%%ebx\n\t" 127 "cpuid\n\t" 128 "movl\t%%ebx,%1\n\t" 129 "popl\t%%ebx" 130 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 131 : "0" (ax), "c" (cx)); 132 } 133 #endif 134 135 static __inline void 136 enable_intr(void) 137 { 138 __asm __volatile("sti"); 139 } 140 141 static __inline void 142 cpu_monitor(const void *addr, u_long extensions, u_int hints) 143 { 144 __asm __volatile("monitor" 145 : : "a" (addr), "c" (extensions), "d" (hints)); 146 } 147 148 static __inline void 149 cpu_mwait(u_long extensions, u_int hints) 150 { 151 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 152 } 153 154 static __inline void 155 lfence(void) 156 { 157 __asm __volatile("lfence" : : : "memory"); 158 } 159 160 static __inline void 161 mfence(void) 162 { 163 __asm __volatile("mfence" : : : "memory"); 164 } 165 166 static __inline void 167 sfence(void) 168 { 169 __asm __volatile("sfence" : : : "memory"); 170 } 171 172 static __inline void 173 halt(void) 174 { 175 __asm __volatile("hlt"); 176 } 177 178 static __inline u_char 179 inb(u_int port) 180 { 181 u_char data; 182 183 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 184 return (data); 185 } 186 187 static __inline u_int 188 inl(u_int port) 189 { 190 u_int data; 191 192 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 193 return (data); 194 } 195 196 static __inline void 197 insb(u_int port, void *addr, size_t count) 198 { 199 __asm __volatile("cld; rep; insb" 200 : "+D" (addr), "+c" (count) 201 : "d" (port) 202 : "memory"); 203 } 204 205 static __inline void 206 insw(u_int port, void *addr, size_t count) 207 { 208 __asm __volatile("cld; rep; insw" 209 : "+D" (addr), "+c" (count) 210 : "d" (port) 211 : "memory"); 212 } 213 214 static __inline void 215 insl(u_int port, void *addr, size_t count) 216 { 217 __asm __volatile("cld; rep; insl" 218 : "+D" (addr), "+c" (count) 219 : "d" (port) 220 : "memory"); 221 } 222 223 static __inline void 224 invd(void) 225 { 226 __asm __volatile("invd"); 227 } 228 229 static __inline u_short 230 inw(u_int port) 231 { 232 u_short data; 233 234 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 235 return (data); 236 } 237 238 static __inline void 239 outb(u_int port, u_char data) 240 { 241 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 242 } 243 244 static __inline void 245 outl(u_int port, u_int data) 246 { 247 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 248 } 249 250 static __inline void 251 outsb(u_int port, const void *addr, size_t count) 252 { 253 __asm __volatile("cld; rep; outsb" 254 : "+S" (addr), "+c" (count) 255 : "d" (port)); 256 } 257 258 static __inline void 259 outsw(u_int port, const void *addr, size_t count) 260 { 261 __asm __volatile("cld; rep; outsw" 262 : "+S" (addr), "+c" (count) 263 : "d" (port)); 264 } 265 266 static __inline void 267 outsl(u_int port, const void *addr, size_t count) 268 { 269 __asm __volatile("cld; rep; outsl" 270 : "+S" (addr), "+c" (count) 271 : "d" (port)); 272 } 273 274 static __inline void 275 outw(u_int port, u_short data) 276 { 277 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 278 } 279 280 static __inline void 281 ia32_pause(void) 282 { 283 __asm __volatile("pause"); 284 } 285 286 static __inline u_int 287 read_eflags(void) 288 { 289 u_int ef; 290 291 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 292 return (ef); 293 } 294 295 static __inline uint64_t 296 rdmsr(u_int msr) 297 { 298 uint64_t rv; 299 300 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 301 return (rv); 302 } 303 304 static __inline uint32_t 305 rdmsr32(u_int msr) 306 { 307 uint32_t low; 308 309 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 310 return (low); 311 } 312 313 static __inline uint64_t 314 rdpmc(u_int pmc) 315 { 316 uint64_t rv; 317 318 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 319 return (rv); 320 } 321 322 static __inline uint64_t 323 rdtsc(void) 324 { 325 uint64_t rv; 326 327 __asm __volatile("rdtsc" : "=A" (rv)); 328 return (rv); 329 } 330 331 static __inline uint64_t 332 rdtsc_ordered_lfence(void) 333 { 334 lfence(); 335 return (rdtsc()); 336 } 337 338 static __inline uint64_t 339 rdtsc_ordered_mfence(void) 340 { 341 mfence(); 342 return (rdtsc()); 343 } 344 345 static __inline uint64_t 346 rdtscp(void) 347 { 348 uint64_t rv; 349 350 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx"); 351 return (rv); 352 } 353 354 static __inline uint64_t 355 rdtscp_aux(uint32_t *aux) 356 { 357 uint64_t rv; 358 359 __asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux)); 360 return (rv); 361 } 362 363 static __inline uint32_t 364 rdtsc32(void) 365 { 366 uint32_t rv; 367 368 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 369 return (rv); 370 } 371 372 static __inline uint32_t 373 rdtscp32(void) 374 { 375 uint32_t rv; 376 377 __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx"); 378 return (rv); 379 } 380 381 static __inline void 382 wbinvd(void) 383 { 384 __asm __volatile("wbinvd"); 385 } 386 387 static __inline void 388 write_eflags(u_int ef) 389 { 390 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 391 } 392 393 static __inline void 394 wrmsr(u_int msr, uint64_t newval) 395 { 396 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 397 } 398 399 static __inline void 400 load_cr0(u_int data) 401 { 402 403 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 404 } 405 406 static __inline u_int 407 rcr0(void) 408 { 409 u_int data; 410 411 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 412 return (data); 413 } 414 415 static __inline u_int 416 rcr2(void) 417 { 418 u_int data; 419 420 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 421 return (data); 422 } 423 424 static __inline void 425 load_cr3(u_int data) 426 { 427 428 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 429 } 430 431 static __inline u_int 432 rcr3(void) 433 { 434 u_int data; 435 436 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 437 return (data); 438 } 439 440 static __inline void 441 load_cr4(u_int data) 442 { 443 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 444 } 445 446 static __inline u_int 447 rcr4(void) 448 { 449 u_int data; 450 451 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 452 return (data); 453 } 454 455 static __inline uint64_t 456 rxcr(u_int reg) 457 { 458 u_int low, high; 459 460 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 461 return (low | ((uint64_t)high << 32)); 462 } 463 464 static __inline void 465 load_xcr(u_int reg, uint64_t val) 466 { 467 u_int low, high; 468 469 low = val; 470 high = val >> 32; 471 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 472 } 473 474 /* 475 * Global TLB flush (except for thise for pages marked PG_G) 476 */ 477 static __inline void 478 invltlb(void) 479 { 480 481 load_cr3(rcr3()); 482 } 483 484 /* 485 * TLB flush for an individual page (even if it has PG_G). 486 * Only works on 486+ CPUs (i386 does not have PG_G). 487 */ 488 static __inline void 489 invlpg(u_int addr) 490 { 491 492 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 493 } 494 495 static __inline u_short 496 rfs(void) 497 { 498 u_short sel; 499 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 500 return (sel); 501 } 502 503 static __inline uint64_t 504 rgdt(void) 505 { 506 uint64_t gdtr; 507 __asm __volatile("sgdt %0" : "=m" (gdtr)); 508 return (gdtr); 509 } 510 511 static __inline u_short 512 rgs(void) 513 { 514 u_short sel; 515 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 516 return (sel); 517 } 518 519 static __inline uint64_t 520 ridt(void) 521 { 522 uint64_t idtr; 523 __asm __volatile("sidt %0" : "=m" (idtr)); 524 return (idtr); 525 } 526 527 static __inline u_short 528 rldt(void) 529 { 530 u_short ldtr; 531 __asm __volatile("sldt %0" : "=g" (ldtr)); 532 return (ldtr); 533 } 534 535 static __inline u_short 536 rss(void) 537 { 538 u_short sel; 539 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 540 return (sel); 541 } 542 543 static __inline u_short 544 rtr(void) 545 { 546 u_short tr; 547 __asm __volatile("str %0" : "=g" (tr)); 548 return (tr); 549 } 550 551 static __inline void 552 load_fs(u_short sel) 553 { 554 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 555 } 556 557 static __inline void 558 load_gs(u_short sel) 559 { 560 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 561 } 562 563 static __inline void 564 lidt(struct region_descriptor *addr) 565 { 566 __asm __volatile("lidt (%0)" : : "r" (addr)); 567 } 568 569 static __inline void 570 lldt(u_short sel) 571 { 572 __asm __volatile("lldt %0" : : "r" (sel)); 573 } 574 575 static __inline void 576 ltr(u_short sel) 577 { 578 __asm __volatile("ltr %0" : : "r" (sel)); 579 } 580 581 static __inline u_int 582 rdr0(void) 583 { 584 u_int data; 585 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 586 return (data); 587 } 588 589 static __inline void 590 load_dr0(u_int dr0) 591 { 592 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 593 } 594 595 static __inline u_int 596 rdr1(void) 597 { 598 u_int data; 599 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 600 return (data); 601 } 602 603 static __inline void 604 load_dr1(u_int dr1) 605 { 606 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 607 } 608 609 static __inline u_int 610 rdr2(void) 611 { 612 u_int data; 613 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 614 return (data); 615 } 616 617 static __inline void 618 load_dr2(u_int dr2) 619 { 620 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 621 } 622 623 static __inline u_int 624 rdr3(void) 625 { 626 u_int data; 627 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 628 return (data); 629 } 630 631 static __inline void 632 load_dr3(u_int dr3) 633 { 634 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 635 } 636 637 static __inline u_int 638 rdr6(void) 639 { 640 u_int data; 641 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 642 return (data); 643 } 644 645 static __inline void 646 load_dr6(u_int dr6) 647 { 648 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 649 } 650 651 static __inline u_int 652 rdr7(void) 653 { 654 u_int data; 655 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 656 return (data); 657 } 658 659 static __inline void 660 load_dr7(u_int dr7) 661 { 662 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 663 } 664 665 static __inline u_char 666 read_cyrix_reg(u_char reg) 667 { 668 outb(0x22, reg); 669 return inb(0x23); 670 } 671 672 static __inline void 673 write_cyrix_reg(u_char reg, u_char data) 674 { 675 outb(0x22, reg); 676 outb(0x23, data); 677 } 678 679 static __inline register_t 680 intr_disable(void) 681 { 682 register_t eflags; 683 684 eflags = read_eflags(); 685 disable_intr(); 686 return (eflags); 687 } 688 689 static __inline void 690 intr_restore(register_t eflags) 691 { 692 write_eflags(eflags); 693 } 694 695 static __inline uint32_t 696 rdpkru(void) 697 { 698 uint32_t res; 699 700 __asm __volatile("rdpkru" : "=a" (res) : "c" (0) : "edx"); 701 return (res); 702 } 703 704 static __inline void 705 wrpkru(uint32_t mask) 706 { 707 708 __asm __volatile("wrpkru" : : "a" (mask), "c" (0), "d" (0)); 709 } 710 711 void reset_dbregs(void); 712 713 #ifdef _KERNEL 714 int rdmsr_safe(u_int msr, uint64_t *val); 715 int wrmsr_safe(u_int msr, uint64_t newval); 716 #endif 717 718 #endif /* !_MACHINE_CPUFUNC_H_ */ 719