1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1993 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38 #ifndef _MACHINE_CPUFUNC_H_
39 #define _MACHINE_CPUFUNC_H_
40
41 struct region_descriptor;
42
43 #define readb(va) (*(volatile uint8_t *) (va))
44 #define readw(va) (*(volatile uint16_t *) (va))
45 #define readl(va) (*(volatile uint32_t *) (va))
46
47 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
48 #define writew(va, d) (*(volatile uint16_t *) (va) = (d))
49 #define writel(va, d) (*(volatile uint32_t *) (va) = (d))
50
51 static __inline void
breakpoint(void)52 breakpoint(void)
53 {
54 __asm __volatile("int $3");
55 }
56
57 static __inline __pure2 u_int
bsfl(u_int mask)58 bsfl(u_int mask)
59 {
60 u_int result;
61
62 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
63 return (result);
64 }
65
66 static __inline void
clflush(u_long addr)67 clflush(u_long addr)
68 {
69
70 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
71 }
72
73 static __inline void
clflushopt(u_long addr)74 clflushopt(u_long addr)
75 {
76
77 __asm __volatile("clflushopt %0" : : "m" (*(char *)addr));
78 }
79
80 static __inline void
clts(void)81 clts(void)
82 {
83
84 __asm __volatile("clts");
85 }
86
87 static __inline void
disable_intr(void)88 disable_intr(void)
89 {
90 __asm __volatile("cli" : : : "memory");
91 }
92
93 #ifdef _KERNEL
94 static __inline void
cpuid_count(u_int ax,u_int cx,u_int * p)95 cpuid_count(u_int ax, u_int cx, u_int *p)
96 {
97 __asm __volatile("cpuid"
98 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
99 : "0" (ax), "c" (cx));
100 }
101 #else
102 static __inline void
cpuid_count(u_int ax,u_int cx,u_int * p)103 cpuid_count(u_int ax, u_int cx, u_int *p)
104 {
105 __asm __volatile(
106 "pushl\t%%ebx\n\t"
107 "cpuid\n\t"
108 "movl\t%%ebx,%1\n\t"
109 "popl\t%%ebx"
110 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
111 : "0" (ax), "c" (cx));
112 }
113 #endif
114
115 static __inline void
do_cpuid(u_int ax,u_int * p)116 do_cpuid(u_int ax, u_int *p)
117 {
118 cpuid_count(ax, 0, p);
119 }
120
121 static __inline void
enable_intr(void)122 enable_intr(void)
123 {
124 __asm __volatile("sti");
125 }
126
127 static __inline void
cpu_monitor(const void * addr,u_long extensions,u_int hints)128 cpu_monitor(const void *addr, u_long extensions, u_int hints)
129 {
130 __asm __volatile("monitor"
131 : : "a" (addr), "c" (extensions), "d" (hints));
132 }
133
134 static __inline void
cpu_mwait(u_long extensions,u_int hints)135 cpu_mwait(u_long extensions, u_int hints)
136 {
137 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
138 }
139
140 static __inline void
lfence(void)141 lfence(void)
142 {
143 __asm __volatile("lfence" : : : "memory");
144 }
145
146 static __inline void
mfence(void)147 mfence(void)
148 {
149 __asm __volatile("mfence" : : : "memory");
150 }
151
152 static __inline void
sfence(void)153 sfence(void)
154 {
155 __asm __volatile("sfence" : : : "memory");
156 }
157
158 static __inline void
halt(void)159 halt(void)
160 {
161 __asm __volatile("hlt");
162 }
163
164 static __inline u_char
inb(u_int port)165 inb(u_int port)
166 {
167 u_char data;
168
169 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
170 return (data);
171 }
172
173 static __inline u_int
inl(u_int port)174 inl(u_int port)
175 {
176 u_int data;
177
178 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
179 return (data);
180 }
181
182 static __inline void
insb(u_int port,void * addr,size_t count)183 insb(u_int port, void *addr, size_t count)
184 {
185 __asm __volatile("cld; rep; insb"
186 : "+D" (addr), "+c" (count)
187 : "d" (port)
188 : "memory");
189 }
190
191 static __inline void
insw(u_int port,void * addr,size_t count)192 insw(u_int port, void *addr, size_t count)
193 {
194 __asm __volatile("cld; rep; insw"
195 : "+D" (addr), "+c" (count)
196 : "d" (port)
197 : "memory");
198 }
199
200 static __inline void
insl(u_int port,void * addr,size_t count)201 insl(u_int port, void *addr, size_t count)
202 {
203 __asm __volatile("cld; rep; insl"
204 : "+D" (addr), "+c" (count)
205 : "d" (port)
206 : "memory");
207 }
208
209 static __inline void
invd(void)210 invd(void)
211 {
212 __asm __volatile("invd");
213 }
214
215 static __inline u_short
inw(u_int port)216 inw(u_int port)
217 {
218 u_short data;
219
220 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
221 return (data);
222 }
223
224 static __inline void
outb(u_int port,u_char data)225 outb(u_int port, u_char data)
226 {
227 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
228 }
229
230 static __inline void
outl(u_int port,u_int data)231 outl(u_int port, u_int data)
232 {
233 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
234 }
235
236 static __inline void
outsb(u_int port,const void * addr,size_t count)237 outsb(u_int port, const void *addr, size_t count)
238 {
239 __asm __volatile("cld; rep; outsb"
240 : "+S" (addr), "+c" (count)
241 : "d" (port));
242 }
243
244 static __inline void
outsw(u_int port,const void * addr,size_t count)245 outsw(u_int port, const void *addr, size_t count)
246 {
247 __asm __volatile("cld; rep; outsw"
248 : "+S" (addr), "+c" (count)
249 : "d" (port));
250 }
251
252 static __inline void
outsl(u_int port,const void * addr,size_t count)253 outsl(u_int port, const void *addr, size_t count)
254 {
255 __asm __volatile("cld; rep; outsl"
256 : "+S" (addr), "+c" (count)
257 : "d" (port));
258 }
259
260 static __inline void
outw(u_int port,u_short data)261 outw(u_int port, u_short data)
262 {
263 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
264 }
265
266 static __inline void
ia32_pause(void)267 ia32_pause(void)
268 {
269 __asm __volatile("pause");
270 }
271
272 static __inline u_int
read_eflags(void)273 read_eflags(void)
274 {
275 u_int ef;
276
277 __asm __volatile("pushfl; popl %0" : "=r" (ef));
278 return (ef);
279 }
280
281 static __inline uint64_t
rdmsr(u_int msr)282 rdmsr(u_int msr)
283 {
284 uint64_t rv;
285
286 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
287 return (rv);
288 }
289
290 static __inline uint32_t
rdmsr32(u_int msr)291 rdmsr32(u_int msr)
292 {
293 uint32_t low;
294
295 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
296 return (low);
297 }
298
299 static __inline uint64_t
rdpmc(u_int pmc)300 rdpmc(u_int pmc)
301 {
302 uint64_t rv;
303
304 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
305 return (rv);
306 }
307
308 static __inline uint64_t
rdtsc(void)309 rdtsc(void)
310 {
311 uint64_t rv;
312
313 __asm __volatile("rdtsc" : "=A" (rv));
314 return (rv);
315 }
316
317 static __inline uint64_t
rdtsc_ordered_lfence(void)318 rdtsc_ordered_lfence(void)
319 {
320 lfence();
321 return (rdtsc());
322 }
323
324 static __inline uint64_t
rdtsc_ordered_mfence(void)325 rdtsc_ordered_mfence(void)
326 {
327 mfence();
328 return (rdtsc());
329 }
330
331 static __inline uint64_t
rdtscp(void)332 rdtscp(void)
333 {
334 uint64_t rv;
335
336 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx");
337 return (rv);
338 }
339
340 static __inline uint64_t
rdtscp_aux(uint32_t * aux)341 rdtscp_aux(uint32_t *aux)
342 {
343 uint64_t rv;
344
345 __asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux));
346 return (rv);
347 }
348
349 static __inline uint32_t
rdtsc32(void)350 rdtsc32(void)
351 {
352 uint32_t rv;
353
354 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
355 return (rv);
356 }
357
358 static __inline uint32_t
rdtscp32(void)359 rdtscp32(void)
360 {
361 uint32_t rv;
362
363 __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx");
364 return (rv);
365 }
366
367 static __inline void
wbinvd(void)368 wbinvd(void)
369 {
370 __asm __volatile("wbinvd");
371 }
372
373 static __inline void
write_eflags(u_int ef)374 write_eflags(u_int ef)
375 {
376 __asm __volatile("pushl %0; popfl" : : "r" (ef));
377 }
378
379 static __inline void
wrmsr(u_int msr,uint64_t newval)380 wrmsr(u_int msr, uint64_t newval)
381 {
382 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
383 }
384
385 static __inline void
load_cr0(u_int data)386 load_cr0(u_int data)
387 {
388
389 __asm __volatile("movl %0,%%cr0" : : "r" (data));
390 }
391
392 static __inline u_int
rcr0(void)393 rcr0(void)
394 {
395 u_int data;
396
397 __asm __volatile("movl %%cr0,%0" : "=r" (data));
398 return (data);
399 }
400
401 static __inline u_int
rcr2(void)402 rcr2(void)
403 {
404 u_int data;
405
406 __asm __volatile("movl %%cr2,%0" : "=r" (data));
407 return (data);
408 }
409
410 static __inline void
load_cr3(u_int data)411 load_cr3(u_int data)
412 {
413
414 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
415 }
416
417 static __inline u_int
rcr3(void)418 rcr3(void)
419 {
420 u_int data;
421
422 __asm __volatile("movl %%cr3,%0" : "=r" (data));
423 return (data);
424 }
425
426 static __inline void
load_cr4(u_int data)427 load_cr4(u_int data)
428 {
429 __asm __volatile("movl %0,%%cr4" : : "r" (data));
430 }
431
432 static __inline u_int
rcr4(void)433 rcr4(void)
434 {
435 u_int data;
436
437 __asm __volatile("movl %%cr4,%0" : "=r" (data));
438 return (data);
439 }
440
441 static __inline uint64_t
rxcr(u_int reg)442 rxcr(u_int reg)
443 {
444 u_int low, high;
445
446 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
447 return (low | ((uint64_t)high << 32));
448 }
449
450 static __inline void
load_xcr(u_int reg,uint64_t val)451 load_xcr(u_int reg, uint64_t val)
452 {
453 u_int low, high;
454
455 low = val;
456 high = val >> 32;
457 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
458 }
459
460 /*
461 * Global TLB flush (except for thise for pages marked PG_G)
462 */
463 static __inline void
invltlb(void)464 invltlb(void)
465 {
466
467 load_cr3(rcr3());
468 }
469
470 /*
471 * TLB flush for an individual page (even if it has PG_G).
472 * Only works on 486+ CPUs (i386 does not have PG_G).
473 */
474 static __inline void
invlpg(u_int addr)475 invlpg(u_int addr)
476 {
477
478 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
479 }
480
481 static __inline u_short
rfs(void)482 rfs(void)
483 {
484 u_short sel;
485 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
486 return (sel);
487 }
488
489 static __inline uint64_t
rgdt(void)490 rgdt(void)
491 {
492 uint64_t gdtr;
493 __asm __volatile("sgdt %0" : "=m" (gdtr));
494 return (gdtr);
495 }
496
497 static __inline u_short
rgs(void)498 rgs(void)
499 {
500 u_short sel;
501 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
502 return (sel);
503 }
504
505 static __inline uint64_t
ridt(void)506 ridt(void)
507 {
508 uint64_t idtr;
509 __asm __volatile("sidt %0" : "=m" (idtr));
510 return (idtr);
511 }
512
513 static __inline u_short
rldt(void)514 rldt(void)
515 {
516 u_short ldtr;
517 __asm __volatile("sldt %0" : "=g" (ldtr));
518 return (ldtr);
519 }
520
521 static __inline u_short
rss(void)522 rss(void)
523 {
524 u_short sel;
525 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
526 return (sel);
527 }
528
529 static __inline u_short
rtr(void)530 rtr(void)
531 {
532 u_short tr;
533 __asm __volatile("str %0" : "=g" (tr));
534 return (tr);
535 }
536
537 static __inline void
load_fs(u_short sel)538 load_fs(u_short sel)
539 {
540 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
541 }
542
543 static __inline void
load_gs(u_short sel)544 load_gs(u_short sel)
545 {
546 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
547 }
548
549 static __inline void
lidt(struct region_descriptor * addr)550 lidt(struct region_descriptor *addr)
551 {
552 __asm __volatile("lidt (%0)" : : "r" (addr));
553 }
554
555 static __inline void
lldt(u_short sel)556 lldt(u_short sel)
557 {
558 __asm __volatile("lldt %0" : : "r" (sel));
559 }
560
561 static __inline void
ltr(u_short sel)562 ltr(u_short sel)
563 {
564 __asm __volatile("ltr %0" : : "r" (sel));
565 }
566
567 static __inline u_int
rdr0(void)568 rdr0(void)
569 {
570 u_int data;
571 __asm __volatile("movl %%dr0,%0" : "=r" (data));
572 return (data);
573 }
574
575 static __inline void
load_dr0(u_int dr0)576 load_dr0(u_int dr0)
577 {
578 __asm __volatile("movl %0,%%dr0" : : "r" (dr0));
579 }
580
581 static __inline u_int
rdr1(void)582 rdr1(void)
583 {
584 u_int data;
585 __asm __volatile("movl %%dr1,%0" : "=r" (data));
586 return (data);
587 }
588
589 static __inline void
load_dr1(u_int dr1)590 load_dr1(u_int dr1)
591 {
592 __asm __volatile("movl %0,%%dr1" : : "r" (dr1));
593 }
594
595 static __inline u_int
rdr2(void)596 rdr2(void)
597 {
598 u_int data;
599 __asm __volatile("movl %%dr2,%0" : "=r" (data));
600 return (data);
601 }
602
603 static __inline void
load_dr2(u_int dr2)604 load_dr2(u_int dr2)
605 {
606 __asm __volatile("movl %0,%%dr2" : : "r" (dr2));
607 }
608
609 static __inline u_int
rdr3(void)610 rdr3(void)
611 {
612 u_int data;
613 __asm __volatile("movl %%dr3,%0" : "=r" (data));
614 return (data);
615 }
616
617 static __inline void
load_dr3(u_int dr3)618 load_dr3(u_int dr3)
619 {
620 __asm __volatile("movl %0,%%dr3" : : "r" (dr3));
621 }
622
623 static __inline u_int
rdr6(void)624 rdr6(void)
625 {
626 u_int data;
627 __asm __volatile("movl %%dr6,%0" : "=r" (data));
628 return (data);
629 }
630
631 static __inline void
load_dr6(u_int dr6)632 load_dr6(u_int dr6)
633 {
634 __asm __volatile("movl %0,%%dr6" : : "r" (dr6));
635 }
636
637 static __inline u_int
rdr7(void)638 rdr7(void)
639 {
640 u_int data;
641 __asm __volatile("movl %%dr7,%0" : "=r" (data));
642 return (data);
643 }
644
645 static __inline void
load_dr7(u_int dr7)646 load_dr7(u_int dr7)
647 {
648 __asm __volatile("movl %0,%%dr7" : : "r" (dr7));
649 }
650
651 static __inline u_char
read_cyrix_reg(u_char reg)652 read_cyrix_reg(u_char reg)
653 {
654 outb(0x22, reg);
655 return inb(0x23);
656 }
657
658 static __inline void
write_cyrix_reg(u_char reg,u_char data)659 write_cyrix_reg(u_char reg, u_char data)
660 {
661 outb(0x22, reg);
662 outb(0x23, data);
663 }
664
665 static __inline register_t
intr_disable(void)666 intr_disable(void)
667 {
668 register_t eflags;
669
670 eflags = read_eflags();
671 disable_intr();
672 return (eflags);
673 }
674
675 static __inline void
intr_restore(register_t eflags)676 intr_restore(register_t eflags)
677 {
678 write_eflags(eflags);
679 }
680
681 static __inline uint32_t
rdpkru(void)682 rdpkru(void)
683 {
684 uint32_t res;
685
686 __asm __volatile("rdpkru" : "=a" (res) : "c" (0) : "edx");
687 return (res);
688 }
689
690 static __inline void
wrpkru(uint32_t mask)691 wrpkru(uint32_t mask)
692 {
693
694 __asm __volatile("wrpkru" : : "a" (mask), "c" (0), "d" (0));
695 }
696
697 void reset_dbregs(void);
698
699 #ifdef _KERNEL
700 int rdmsr_safe(u_int msr, uint64_t *val);
701 int wrmsr_safe(u_int msr, uint64_t newval);
702 #endif
703
704 #endif /* !_MACHINE_CPUFUNC_H_ */
705