xref: /freebsd/sys/i386/include/cpufunc.h (revision a0b9e2e854027e6ff61fb075a1309dbc71c42b54)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1993 The Regents of the University of California.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. Neither the name of the University nor the names of its contributors
16  *    may be used to endorse or promote products derived from this software
17  *    without specific prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  *
31  * $FreeBSD$
32  */
33 
34 /*
35  * Functions to provide access to special i386 instructions.
36  * This in included in sys/systm.h, and that file should be
37  * used in preference to this.
38  */
39 
40 #ifndef _MACHINE_CPUFUNC_H_
41 #define	_MACHINE_CPUFUNC_H_
42 
43 #ifndef _SYS_CDEFS_H_
44 #error this file needs sys/cdefs.h as a prerequisite
45 #endif
46 
47 struct region_descriptor;
48 
49 #define readb(va)	(*(volatile uint8_t *) (va))
50 #define readw(va)	(*(volatile uint16_t *) (va))
51 #define readl(va)	(*(volatile uint32_t *) (va))
52 
53 #define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
54 #define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
55 #define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
56 
57 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
58 
59 static __inline void
60 breakpoint(void)
61 {
62 	__asm __volatile("int $3");
63 }
64 
65 static __inline __pure2 u_int
66 bsfl(u_int mask)
67 {
68 	u_int	result;
69 
70 	__asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
71 	return (result);
72 }
73 
74 static __inline __pure2 u_int
75 bsrl(u_int mask)
76 {
77 	u_int	result;
78 
79 	__asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
80 	return (result);
81 }
82 
83 static __inline void
84 clflush(u_long addr)
85 {
86 
87 	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
88 }
89 
90 static __inline void
91 clflushopt(u_long addr)
92 {
93 
94 	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
95 }
96 
97 static __inline void
98 clts(void)
99 {
100 
101 	__asm __volatile("clts");
102 }
103 
104 static __inline void
105 disable_intr(void)
106 {
107 
108 	__asm __volatile("cli" : : : "memory");
109 }
110 
111 #ifdef _KERNEL
112 static __inline void
113 do_cpuid(u_int ax, u_int *p)
114 {
115 	__asm __volatile("cpuid"
116 	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
117 	    :  "0" (ax));
118 }
119 
120 static __inline void
121 cpuid_count(u_int ax, u_int cx, u_int *p)
122 {
123 	__asm __volatile("cpuid"
124 	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
125 	    :  "0" (ax), "c" (cx));
126 }
127 #else
128 static __inline void
129 do_cpuid(u_int ax, u_int *p)
130 {
131 	__asm __volatile(
132 	    "pushl\t%%ebx\n\t"
133 	    "cpuid\n\t"
134 	    "movl\t%%ebx,%1\n\t"
135 	    "popl\t%%ebx"
136 	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
137 	    :  "0" (ax));
138 }
139 
140 static __inline void
141 cpuid_count(u_int ax, u_int cx, u_int *p)
142 {
143 	__asm __volatile(
144 	    "pushl\t%%ebx\n\t"
145 	    "cpuid\n\t"
146 	    "movl\t%%ebx,%1\n\t"
147 	    "popl\t%%ebx"
148 	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
149 	    :  "0" (ax), "c" (cx));
150 }
151 #endif
152 
153 static __inline void
154 enable_intr(void)
155 {
156 
157 	__asm __volatile("sti");
158 }
159 
160 static __inline void
161 cpu_monitor(const void *addr, u_long extensions, u_int hints)
162 {
163 
164 	__asm __volatile("monitor"
165 	    : : "a" (addr), "c" (extensions), "d" (hints));
166 }
167 
168 static __inline void
169 cpu_mwait(u_long extensions, u_int hints)
170 {
171 
172 	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
173 }
174 
175 static __inline void
176 lfence(void)
177 {
178 
179 	__asm __volatile("lfence" : : : "memory");
180 }
181 
182 static __inline void
183 mfence(void)
184 {
185 
186 	__asm __volatile("mfence" : : : "memory");
187 }
188 
189 static __inline void
190 sfence(void)
191 {
192 
193 	__asm __volatile("sfence" : : : "memory");
194 }
195 
196 #ifdef _KERNEL
197 
198 #define	HAVE_INLINE_FFS
199 
200 static __inline __pure2 int
201 ffs(int mask)
202 {
203 	/*
204 	 * Note that gcc-2's builtin ffs would be used if we didn't declare
205 	 * this inline or turn off the builtin.  The builtin is faster but
206 	 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
207 	 * versions.
208 	 */
209 	 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
210 }
211 
212 #define	HAVE_INLINE_FFSL
213 
214 static __inline __pure2 int
215 ffsl(long mask)
216 {
217 	return (ffs((int)mask));
218 }
219 
220 #define	HAVE_INLINE_FLS
221 
222 static __inline __pure2 int
223 fls(int mask)
224 {
225 	return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
226 }
227 
228 #define	HAVE_INLINE_FLSL
229 
230 static __inline __pure2 int
231 flsl(long mask)
232 {
233 	return (fls((int)mask));
234 }
235 
236 #endif /* _KERNEL */
237 
238 static __inline void
239 halt(void)
240 {
241 	__asm __volatile("hlt");
242 }
243 
244 static __inline u_char
245 inb(u_int port)
246 {
247 	u_char	data;
248 
249 	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
250 	return (data);
251 }
252 
253 static __inline u_int
254 inl(u_int port)
255 {
256 	u_int	data;
257 
258 	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
259 	return (data);
260 }
261 
262 static __inline void
263 insb(u_int port, void *addr, size_t count)
264 {
265 	__asm __volatile("cld; rep; insb"
266 			 : "+D" (addr), "+c" (count)
267 			 : "d" (port)
268 			 : "memory");
269 }
270 
271 static __inline void
272 insw(u_int port, void *addr, size_t count)
273 {
274 	__asm __volatile("cld; rep; insw"
275 			 : "+D" (addr), "+c" (count)
276 			 : "d" (port)
277 			 : "memory");
278 }
279 
280 static __inline void
281 insl(u_int port, void *addr, size_t count)
282 {
283 	__asm __volatile("cld; rep; insl"
284 			 : "+D" (addr), "+c" (count)
285 			 : "d" (port)
286 			 : "memory");
287 }
288 
289 static __inline void
290 invd(void)
291 {
292 	__asm __volatile("invd");
293 }
294 
295 static __inline u_short
296 inw(u_int port)
297 {
298 	u_short	data;
299 
300 	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
301 	return (data);
302 }
303 
304 static __inline void
305 outb(u_int port, u_char data)
306 {
307 	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
308 }
309 
310 static __inline void
311 outl(u_int port, u_int data)
312 {
313 	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
314 }
315 
316 static __inline void
317 outsb(u_int port, const void *addr, size_t count)
318 {
319 	__asm __volatile("cld; rep; outsb"
320 			 : "+S" (addr), "+c" (count)
321 			 : "d" (port));
322 }
323 
324 static __inline void
325 outsw(u_int port, const void *addr, size_t count)
326 {
327 	__asm __volatile("cld; rep; outsw"
328 			 : "+S" (addr), "+c" (count)
329 			 : "d" (port));
330 }
331 
332 static __inline void
333 outsl(u_int port, const void *addr, size_t count)
334 {
335 	__asm __volatile("cld; rep; outsl"
336 			 : "+S" (addr), "+c" (count)
337 			 : "d" (port));
338 }
339 
340 static __inline void
341 outw(u_int port, u_short data)
342 {
343 	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
344 }
345 
346 static __inline void
347 ia32_pause(void)
348 {
349 	__asm __volatile("pause");
350 }
351 
352 static __inline u_int
353 read_eflags(void)
354 {
355 	u_int	ef;
356 
357 	__asm __volatile("pushfl; popl %0" : "=r" (ef));
358 	return (ef);
359 }
360 
361 static __inline uint64_t
362 rdmsr(u_int msr)
363 {
364 	uint64_t rv;
365 
366 	__asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
367 	return (rv);
368 }
369 
370 static __inline uint32_t
371 rdmsr32(u_int msr)
372 {
373 	uint32_t low;
374 
375 	__asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
376 	return (low);
377 }
378 
379 static __inline uint64_t
380 rdpmc(u_int pmc)
381 {
382 	uint64_t rv;
383 
384 	__asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
385 	return (rv);
386 }
387 
388 static __inline uint64_t
389 rdtsc(void)
390 {
391 	uint64_t rv;
392 
393 	__asm __volatile("rdtsc" : "=A" (rv));
394 	return (rv);
395 }
396 
397 static __inline uint64_t
398 rdtscp(void)
399 {
400 	uint64_t rv;
401 
402 	__asm __volatile("rdtscp" : "=A" (rv) : : "ecx");
403 	return (rv);
404 }
405 
406 static __inline uint32_t
407 rdtsc32(void)
408 {
409 	uint32_t rv;
410 
411 	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
412 	return (rv);
413 }
414 
415 static __inline void
416 wbinvd(void)
417 {
418 	__asm __volatile("wbinvd");
419 }
420 
421 static __inline void
422 write_eflags(u_int ef)
423 {
424 	__asm __volatile("pushl %0; popfl" : : "r" (ef));
425 }
426 
427 static __inline void
428 wrmsr(u_int msr, uint64_t newval)
429 {
430 	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
431 }
432 
433 static __inline void
434 load_cr0(u_int data)
435 {
436 
437 	__asm __volatile("movl %0,%%cr0" : : "r" (data));
438 }
439 
440 static __inline u_int
441 rcr0(void)
442 {
443 	u_int	data;
444 
445 	__asm __volatile("movl %%cr0,%0" : "=r" (data));
446 	return (data);
447 }
448 
449 static __inline u_int
450 rcr2(void)
451 {
452 	u_int	data;
453 
454 	__asm __volatile("movl %%cr2,%0" : "=r" (data));
455 	return (data);
456 }
457 
458 static __inline void
459 load_cr3(u_int data)
460 {
461 
462 	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
463 }
464 
465 static __inline u_int
466 rcr3(void)
467 {
468 	u_int	data;
469 
470 	__asm __volatile("movl %%cr3,%0" : "=r" (data));
471 	return (data);
472 }
473 
474 static __inline void
475 load_cr4(u_int data)
476 {
477 	__asm __volatile("movl %0,%%cr4" : : "r" (data));
478 }
479 
480 static __inline u_int
481 rcr4(void)
482 {
483 	u_int	data;
484 
485 	__asm __volatile("movl %%cr4,%0" : "=r" (data));
486 	return (data);
487 }
488 
489 static __inline uint64_t
490 rxcr(u_int reg)
491 {
492 	u_int low, high;
493 
494 	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
495 	return (low | ((uint64_t)high << 32));
496 }
497 
498 static __inline void
499 load_xcr(u_int reg, uint64_t val)
500 {
501 	u_int low, high;
502 
503 	low = val;
504 	high = val >> 32;
505 	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
506 }
507 
508 /*
509  * Global TLB flush (except for thise for pages marked PG_G)
510  */
511 static __inline void
512 invltlb(void)
513 {
514 
515 	load_cr3(rcr3());
516 }
517 
518 /*
519  * TLB flush for an individual page (even if it has PG_G).
520  * Only works on 486+ CPUs (i386 does not have PG_G).
521  */
522 static __inline void
523 invlpg(u_int addr)
524 {
525 
526 	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
527 }
528 
529 static __inline u_short
530 rfs(void)
531 {
532 	u_short sel;
533 	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
534 	return (sel);
535 }
536 
537 static __inline uint64_t
538 rgdt(void)
539 {
540 	uint64_t gdtr;
541 	__asm __volatile("sgdt %0" : "=m" (gdtr));
542 	return (gdtr);
543 }
544 
545 static __inline u_short
546 rgs(void)
547 {
548 	u_short sel;
549 	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
550 	return (sel);
551 }
552 
553 static __inline uint64_t
554 ridt(void)
555 {
556 	uint64_t idtr;
557 	__asm __volatile("sidt %0" : "=m" (idtr));
558 	return (idtr);
559 }
560 
561 static __inline u_short
562 rldt(void)
563 {
564 	u_short ldtr;
565 	__asm __volatile("sldt %0" : "=g" (ldtr));
566 	return (ldtr);
567 }
568 
569 static __inline u_short
570 rss(void)
571 {
572 	u_short sel;
573 	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
574 	return (sel);
575 }
576 
577 static __inline u_short
578 rtr(void)
579 {
580 	u_short tr;
581 	__asm __volatile("str %0" : "=g" (tr));
582 	return (tr);
583 }
584 
585 static __inline void
586 load_fs(u_short sel)
587 {
588 	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
589 }
590 
591 static __inline void
592 load_gs(u_short sel)
593 {
594 	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
595 }
596 
597 static __inline void
598 lidt(struct region_descriptor *addr)
599 {
600 	__asm __volatile("lidt (%0)" : : "r" (addr));
601 }
602 
603 static __inline void
604 lldt(u_short sel)
605 {
606 	__asm __volatile("lldt %0" : : "r" (sel));
607 }
608 
609 static __inline void
610 ltr(u_short sel)
611 {
612 	__asm __volatile("ltr %0" : : "r" (sel));
613 }
614 
615 static __inline u_int
616 rdr0(void)
617 {
618 	u_int	data;
619 	__asm __volatile("movl %%dr0,%0" : "=r" (data));
620 	return (data);
621 }
622 
623 static __inline void
624 load_dr0(u_int dr0)
625 {
626 	__asm __volatile("movl %0,%%dr0" : : "r" (dr0));
627 }
628 
629 static __inline u_int
630 rdr1(void)
631 {
632 	u_int	data;
633 	__asm __volatile("movl %%dr1,%0" : "=r" (data));
634 	return (data);
635 }
636 
637 static __inline void
638 load_dr1(u_int dr1)
639 {
640 	__asm __volatile("movl %0,%%dr1" : : "r" (dr1));
641 }
642 
643 static __inline u_int
644 rdr2(void)
645 {
646 	u_int	data;
647 	__asm __volatile("movl %%dr2,%0" : "=r" (data));
648 	return (data);
649 }
650 
651 static __inline void
652 load_dr2(u_int dr2)
653 {
654 	__asm __volatile("movl %0,%%dr2" : : "r" (dr2));
655 }
656 
657 static __inline u_int
658 rdr3(void)
659 {
660 	u_int	data;
661 	__asm __volatile("movl %%dr3,%0" : "=r" (data));
662 	return (data);
663 }
664 
665 static __inline void
666 load_dr3(u_int dr3)
667 {
668 	__asm __volatile("movl %0,%%dr3" : : "r" (dr3));
669 }
670 
671 static __inline u_int
672 rdr6(void)
673 {
674 	u_int	data;
675 	__asm __volatile("movl %%dr6,%0" : "=r" (data));
676 	return (data);
677 }
678 
679 static __inline void
680 load_dr6(u_int dr6)
681 {
682 	__asm __volatile("movl %0,%%dr6" : : "r" (dr6));
683 }
684 
685 static __inline u_int
686 rdr7(void)
687 {
688 	u_int	data;
689 	__asm __volatile("movl %%dr7,%0" : "=r" (data));
690 	return (data);
691 }
692 
693 static __inline void
694 load_dr7(u_int dr7)
695 {
696 	__asm __volatile("movl %0,%%dr7" : : "r" (dr7));
697 }
698 
699 static __inline u_char
700 read_cyrix_reg(u_char reg)
701 {
702 	outb(0x22, reg);
703 	return inb(0x23);
704 }
705 
706 static __inline void
707 write_cyrix_reg(u_char reg, u_char data)
708 {
709 	outb(0x22, reg);
710 	outb(0x23, data);
711 }
712 
713 static __inline register_t
714 intr_disable(void)
715 {
716 	register_t eflags;
717 
718 	eflags = read_eflags();
719 	disable_intr();
720 	return (eflags);
721 }
722 
723 static __inline void
724 intr_restore(register_t eflags)
725 {
726 	write_eflags(eflags);
727 }
728 
729 static __inline uint32_t
730 rdpkru(void)
731 {
732 	uint32_t res;
733 
734 	__asm __volatile("rdpkru" :  "=a" (res) : "c" (0) : "edx");
735 	return (res);
736 }
737 
738 static __inline void
739 wrpkru(uint32_t mask)
740 {
741 
742 	__asm __volatile("wrpkru" :  : "a" (mask),  "c" (0), "d" (0));
743 }
744 
745 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
746 
747 int	breakpoint(void);
748 u_int	bsfl(u_int mask);
749 u_int	bsrl(u_int mask);
750 void	clflush(u_long addr);
751 void	clts(void);
752 void	cpuid_count(u_int ax, u_int cx, u_int *p);
753 void	disable_intr(void);
754 void	do_cpuid(u_int ax, u_int *p);
755 void	enable_intr(void);
756 void	halt(void);
757 void	ia32_pause(void);
758 u_char	inb(u_int port);
759 u_int	inl(u_int port);
760 void	insb(u_int port, void *addr, size_t count);
761 void	insl(u_int port, void *addr, size_t count);
762 void	insw(u_int port, void *addr, size_t count);
763 register_t	intr_disable(void);
764 void	intr_restore(register_t ef);
765 void	invd(void);
766 void	invlpg(u_int addr);
767 void	invltlb(void);
768 u_short	inw(u_int port);
769 void	lidt(struct region_descriptor *addr);
770 void	lldt(u_short sel);
771 void	load_cr0(u_int cr0);
772 void	load_cr3(u_int cr3);
773 void	load_cr4(u_int cr4);
774 void	load_dr0(u_int dr0);
775 void	load_dr1(u_int dr1);
776 void	load_dr2(u_int dr2);
777 void	load_dr3(u_int dr3);
778 void	load_dr6(u_int dr6);
779 void	load_dr7(u_int dr7);
780 void	load_fs(u_short sel);
781 void	load_gs(u_short sel);
782 void	ltr(u_short sel);
783 void	outb(u_int port, u_char data);
784 void	outl(u_int port, u_int data);
785 void	outsb(u_int port, const void *addr, size_t count);
786 void	outsl(u_int port, const void *addr, size_t count);
787 void	outsw(u_int port, const void *addr, size_t count);
788 void	outw(u_int port, u_short data);
789 u_int	rcr0(void);
790 u_int	rcr2(void);
791 u_int	rcr3(void);
792 u_int	rcr4(void);
793 uint64_t rdmsr(u_int msr);
794 uint64_t rdpmc(u_int pmc);
795 u_int	rdr0(void);
796 u_int	rdr1(void);
797 u_int	rdr2(void);
798 u_int	rdr3(void);
799 u_int	rdr6(void);
800 u_int	rdr7(void);
801 uint64_t rdtsc(void);
802 u_char	read_cyrix_reg(u_char reg);
803 u_int	read_eflags(void);
804 u_int	rfs(void);
805 uint64_t rgdt(void);
806 u_int	rgs(void);
807 uint64_t ridt(void);
808 u_short	rldt(void);
809 u_short	rtr(void);
810 void	wbinvd(void);
811 void	write_cyrix_reg(u_char reg, u_char data);
812 void	write_eflags(u_int ef);
813 void	wrmsr(u_int msr, uint64_t newval);
814 
815 #endif	/* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
816 
817 void    reset_dbregs(void);
818 
819 #ifdef _KERNEL
820 int	rdmsr_safe(u_int msr, uint64_t *val);
821 int	wrmsr_safe(u_int msr, uint64_t newval);
822 #endif
823 
824 #endif /* !_MACHINE_CPUFUNC_H_ */
825