xref: /freebsd/sys/i386/include/cpufunc.h (revision 7ef62cebc2f965b0f640263e179276928885e33d)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1993 The Regents of the University of California.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. Neither the name of the University nor the names of its contributors
16  *    may be used to endorse or promote products derived from this software
17  *    without specific prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  *
31  * $FreeBSD$
32  */
33 
34 /*
35  * Functions to provide access to special i386 instructions.
36  * This in included in sys/systm.h, and that file should be
37  * used in preference to this.
38  */
39 
40 #ifndef _MACHINE_CPUFUNC_H_
41 #define	_MACHINE_CPUFUNC_H_
42 
43 struct region_descriptor;
44 
45 #define readb(va)	(*(volatile uint8_t *) (va))
46 #define readw(va)	(*(volatile uint16_t *) (va))
47 #define readl(va)	(*(volatile uint32_t *) (va))
48 
49 #define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
50 #define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
51 #define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
52 
53 static __inline void
54 breakpoint(void)
55 {
56 	__asm __volatile("int $3");
57 }
58 
59 static __inline __pure2 u_int
60 bsfl(u_int mask)
61 {
62 	u_int	result;
63 
64 	__asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
65 	return (result);
66 }
67 
68 static __inline __pure2 u_int
69 bsrl(u_int mask)
70 {
71 	u_int	result;
72 
73 	__asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
74 	return (result);
75 }
76 
77 static __inline void
78 clflush(u_long addr)
79 {
80 
81 	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
82 }
83 
84 static __inline void
85 clflushopt(u_long addr)
86 {
87 
88 	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
89 }
90 
91 static __inline void
92 clts(void)
93 {
94 
95 	__asm __volatile("clts");
96 }
97 
98 static __inline void
99 disable_intr(void)
100 {
101 	__asm __volatile("cli" : : : "memory");
102 }
103 
104 #ifdef _KERNEL
105 static __inline void
106 do_cpuid(u_int ax, u_int *p)
107 {
108 	__asm __volatile("cpuid"
109 	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
110 	    :  "0" (ax));
111 }
112 
113 static __inline void
114 cpuid_count(u_int ax, u_int cx, u_int *p)
115 {
116 	__asm __volatile("cpuid"
117 	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
118 	    :  "0" (ax), "c" (cx));
119 }
120 #else
121 static __inline void
122 do_cpuid(u_int ax, u_int *p)
123 {
124 	__asm __volatile(
125 	    "pushl\t%%ebx\n\t"
126 	    "cpuid\n\t"
127 	    "movl\t%%ebx,%1\n\t"
128 	    "popl\t%%ebx"
129 	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
130 	    :  "0" (ax));
131 }
132 
133 static __inline void
134 cpuid_count(u_int ax, u_int cx, u_int *p)
135 {
136 	__asm __volatile(
137 	    "pushl\t%%ebx\n\t"
138 	    "cpuid\n\t"
139 	    "movl\t%%ebx,%1\n\t"
140 	    "popl\t%%ebx"
141 	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
142 	    :  "0" (ax), "c" (cx));
143 }
144 #endif
145 
146 static __inline void
147 enable_intr(void)
148 {
149 	__asm __volatile("sti");
150 }
151 
152 static __inline void
153 cpu_monitor(const void *addr, u_long extensions, u_int hints)
154 {
155 	__asm __volatile("monitor"
156 	    : : "a" (addr), "c" (extensions), "d" (hints));
157 }
158 
159 static __inline void
160 cpu_mwait(u_long extensions, u_int hints)
161 {
162 	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
163 }
164 
165 static __inline void
166 lfence(void)
167 {
168 	__asm __volatile("lfence" : : : "memory");
169 }
170 
171 static __inline void
172 mfence(void)
173 {
174 	__asm __volatile("mfence" : : : "memory");
175 }
176 
177 static __inline void
178 sfence(void)
179 {
180 	__asm __volatile("sfence" : : : "memory");
181 }
182 
183 static __inline void
184 halt(void)
185 {
186 	__asm __volatile("hlt");
187 }
188 
189 static __inline u_char
190 inb(u_int port)
191 {
192 	u_char	data;
193 
194 	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
195 	return (data);
196 }
197 
198 static __inline u_int
199 inl(u_int port)
200 {
201 	u_int	data;
202 
203 	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
204 	return (data);
205 }
206 
207 static __inline void
208 insb(u_int port, void *addr, size_t count)
209 {
210 	__asm __volatile("cld; rep; insb"
211 			 : "+D" (addr), "+c" (count)
212 			 : "d" (port)
213 			 : "memory");
214 }
215 
216 static __inline void
217 insw(u_int port, void *addr, size_t count)
218 {
219 	__asm __volatile("cld; rep; insw"
220 			 : "+D" (addr), "+c" (count)
221 			 : "d" (port)
222 			 : "memory");
223 }
224 
225 static __inline void
226 insl(u_int port, void *addr, size_t count)
227 {
228 	__asm __volatile("cld; rep; insl"
229 			 : "+D" (addr), "+c" (count)
230 			 : "d" (port)
231 			 : "memory");
232 }
233 
234 static __inline void
235 invd(void)
236 {
237 	__asm __volatile("invd");
238 }
239 
240 static __inline u_short
241 inw(u_int port)
242 {
243 	u_short	data;
244 
245 	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
246 	return (data);
247 }
248 
249 static __inline void
250 outb(u_int port, u_char data)
251 {
252 	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
253 }
254 
255 static __inline void
256 outl(u_int port, u_int data)
257 {
258 	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
259 }
260 
261 static __inline void
262 outsb(u_int port, const void *addr, size_t count)
263 {
264 	__asm __volatile("cld; rep; outsb"
265 			 : "+S" (addr), "+c" (count)
266 			 : "d" (port));
267 }
268 
269 static __inline void
270 outsw(u_int port, const void *addr, size_t count)
271 {
272 	__asm __volatile("cld; rep; outsw"
273 			 : "+S" (addr), "+c" (count)
274 			 : "d" (port));
275 }
276 
277 static __inline void
278 outsl(u_int port, const void *addr, size_t count)
279 {
280 	__asm __volatile("cld; rep; outsl"
281 			 : "+S" (addr), "+c" (count)
282 			 : "d" (port));
283 }
284 
285 static __inline void
286 outw(u_int port, u_short data)
287 {
288 	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
289 }
290 
291 static __inline void
292 ia32_pause(void)
293 {
294 	__asm __volatile("pause");
295 }
296 
297 static __inline u_int
298 read_eflags(void)
299 {
300 	u_int	ef;
301 
302 	__asm __volatile("pushfl; popl %0" : "=r" (ef));
303 	return (ef);
304 }
305 
306 static __inline uint64_t
307 rdmsr(u_int msr)
308 {
309 	uint64_t rv;
310 
311 	__asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
312 	return (rv);
313 }
314 
315 static __inline uint32_t
316 rdmsr32(u_int msr)
317 {
318 	uint32_t low;
319 
320 	__asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
321 	return (low);
322 }
323 
324 static __inline uint64_t
325 rdpmc(u_int pmc)
326 {
327 	uint64_t rv;
328 
329 	__asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
330 	return (rv);
331 }
332 
333 static __inline uint64_t
334 rdtsc(void)
335 {
336 	uint64_t rv;
337 
338 	__asm __volatile("rdtsc" : "=A" (rv));
339 	return (rv);
340 }
341 
342 static __inline uint64_t
343 rdtsc_ordered_lfence(void)
344 {
345 	lfence();
346 	return (rdtsc());
347 }
348 
349 static __inline uint64_t
350 rdtsc_ordered_mfence(void)
351 {
352 	mfence();
353 	return (rdtsc());
354 }
355 
356 static __inline uint64_t
357 rdtscp(void)
358 {
359 	uint64_t rv;
360 
361 	__asm __volatile("rdtscp" : "=A" (rv) : : "ecx");
362 	return (rv);
363 }
364 
365 static __inline uint64_t
366 rdtscp_aux(uint32_t *aux)
367 {
368 	uint64_t rv;
369 
370 	__asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux));
371 	return (rv);
372 }
373 
374 static __inline uint32_t
375 rdtsc32(void)
376 {
377 	uint32_t rv;
378 
379 	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
380 	return (rv);
381 }
382 
383 static __inline uint32_t
384 rdtscp32(void)
385 {
386 	uint32_t rv;
387 
388 	__asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx");
389 	return (rv);
390 }
391 
392 static __inline void
393 wbinvd(void)
394 {
395 	__asm __volatile("wbinvd");
396 }
397 
398 static __inline void
399 write_eflags(u_int ef)
400 {
401 	__asm __volatile("pushl %0; popfl" : : "r" (ef));
402 }
403 
404 static __inline void
405 wrmsr(u_int msr, uint64_t newval)
406 {
407 	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
408 }
409 
410 static __inline void
411 load_cr0(u_int data)
412 {
413 
414 	__asm __volatile("movl %0,%%cr0" : : "r" (data));
415 }
416 
417 static __inline u_int
418 rcr0(void)
419 {
420 	u_int	data;
421 
422 	__asm __volatile("movl %%cr0,%0" : "=r" (data));
423 	return (data);
424 }
425 
426 static __inline u_int
427 rcr2(void)
428 {
429 	u_int	data;
430 
431 	__asm __volatile("movl %%cr2,%0" : "=r" (data));
432 	return (data);
433 }
434 
435 static __inline void
436 load_cr3(u_int data)
437 {
438 
439 	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
440 }
441 
442 static __inline u_int
443 rcr3(void)
444 {
445 	u_int	data;
446 
447 	__asm __volatile("movl %%cr3,%0" : "=r" (data));
448 	return (data);
449 }
450 
451 static __inline void
452 load_cr4(u_int data)
453 {
454 	__asm __volatile("movl %0,%%cr4" : : "r" (data));
455 }
456 
457 static __inline u_int
458 rcr4(void)
459 {
460 	u_int	data;
461 
462 	__asm __volatile("movl %%cr4,%0" : "=r" (data));
463 	return (data);
464 }
465 
466 static __inline uint64_t
467 rxcr(u_int reg)
468 {
469 	u_int low, high;
470 
471 	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
472 	return (low | ((uint64_t)high << 32));
473 }
474 
475 static __inline void
476 load_xcr(u_int reg, uint64_t val)
477 {
478 	u_int low, high;
479 
480 	low = val;
481 	high = val >> 32;
482 	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
483 }
484 
485 /*
486  * Global TLB flush (except for thise for pages marked PG_G)
487  */
488 static __inline void
489 invltlb(void)
490 {
491 
492 	load_cr3(rcr3());
493 }
494 
495 /*
496  * TLB flush for an individual page (even if it has PG_G).
497  * Only works on 486+ CPUs (i386 does not have PG_G).
498  */
499 static __inline void
500 invlpg(u_int addr)
501 {
502 
503 	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
504 }
505 
506 static __inline u_short
507 rfs(void)
508 {
509 	u_short sel;
510 	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
511 	return (sel);
512 }
513 
514 static __inline uint64_t
515 rgdt(void)
516 {
517 	uint64_t gdtr;
518 	__asm __volatile("sgdt %0" : "=m" (gdtr));
519 	return (gdtr);
520 }
521 
522 static __inline u_short
523 rgs(void)
524 {
525 	u_short sel;
526 	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
527 	return (sel);
528 }
529 
530 static __inline uint64_t
531 ridt(void)
532 {
533 	uint64_t idtr;
534 	__asm __volatile("sidt %0" : "=m" (idtr));
535 	return (idtr);
536 }
537 
538 static __inline u_short
539 rldt(void)
540 {
541 	u_short ldtr;
542 	__asm __volatile("sldt %0" : "=g" (ldtr));
543 	return (ldtr);
544 }
545 
546 static __inline u_short
547 rss(void)
548 {
549 	u_short sel;
550 	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
551 	return (sel);
552 }
553 
554 static __inline u_short
555 rtr(void)
556 {
557 	u_short tr;
558 	__asm __volatile("str %0" : "=g" (tr));
559 	return (tr);
560 }
561 
562 static __inline void
563 load_fs(u_short sel)
564 {
565 	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
566 }
567 
568 static __inline void
569 load_gs(u_short sel)
570 {
571 	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
572 }
573 
574 static __inline void
575 lidt(struct region_descriptor *addr)
576 {
577 	__asm __volatile("lidt (%0)" : : "r" (addr));
578 }
579 
580 static __inline void
581 lldt(u_short sel)
582 {
583 	__asm __volatile("lldt %0" : : "r" (sel));
584 }
585 
586 static __inline void
587 ltr(u_short sel)
588 {
589 	__asm __volatile("ltr %0" : : "r" (sel));
590 }
591 
592 static __inline u_int
593 rdr0(void)
594 {
595 	u_int	data;
596 	__asm __volatile("movl %%dr0,%0" : "=r" (data));
597 	return (data);
598 }
599 
600 static __inline void
601 load_dr0(u_int dr0)
602 {
603 	__asm __volatile("movl %0,%%dr0" : : "r" (dr0));
604 }
605 
606 static __inline u_int
607 rdr1(void)
608 {
609 	u_int	data;
610 	__asm __volatile("movl %%dr1,%0" : "=r" (data));
611 	return (data);
612 }
613 
614 static __inline void
615 load_dr1(u_int dr1)
616 {
617 	__asm __volatile("movl %0,%%dr1" : : "r" (dr1));
618 }
619 
620 static __inline u_int
621 rdr2(void)
622 {
623 	u_int	data;
624 	__asm __volatile("movl %%dr2,%0" : "=r" (data));
625 	return (data);
626 }
627 
628 static __inline void
629 load_dr2(u_int dr2)
630 {
631 	__asm __volatile("movl %0,%%dr2" : : "r" (dr2));
632 }
633 
634 static __inline u_int
635 rdr3(void)
636 {
637 	u_int	data;
638 	__asm __volatile("movl %%dr3,%0" : "=r" (data));
639 	return (data);
640 }
641 
642 static __inline void
643 load_dr3(u_int dr3)
644 {
645 	__asm __volatile("movl %0,%%dr3" : : "r" (dr3));
646 }
647 
648 static __inline u_int
649 rdr6(void)
650 {
651 	u_int	data;
652 	__asm __volatile("movl %%dr6,%0" : "=r" (data));
653 	return (data);
654 }
655 
656 static __inline void
657 load_dr6(u_int dr6)
658 {
659 	__asm __volatile("movl %0,%%dr6" : : "r" (dr6));
660 }
661 
662 static __inline u_int
663 rdr7(void)
664 {
665 	u_int	data;
666 	__asm __volatile("movl %%dr7,%0" : "=r" (data));
667 	return (data);
668 }
669 
670 static __inline void
671 load_dr7(u_int dr7)
672 {
673 	__asm __volatile("movl %0,%%dr7" : : "r" (dr7));
674 }
675 
676 static __inline u_char
677 read_cyrix_reg(u_char reg)
678 {
679 	outb(0x22, reg);
680 	return inb(0x23);
681 }
682 
683 static __inline void
684 write_cyrix_reg(u_char reg, u_char data)
685 {
686 	outb(0x22, reg);
687 	outb(0x23, data);
688 }
689 
690 static __inline register_t
691 intr_disable(void)
692 {
693 	register_t eflags;
694 
695 	eflags = read_eflags();
696 	disable_intr();
697 	return (eflags);
698 }
699 
700 static __inline void
701 intr_restore(register_t eflags)
702 {
703 	write_eflags(eflags);
704 }
705 
706 static __inline uint32_t
707 rdpkru(void)
708 {
709 	uint32_t res;
710 
711 	__asm __volatile("rdpkru" :  "=a" (res) : "c" (0) : "edx");
712 	return (res);
713 }
714 
715 static __inline void
716 wrpkru(uint32_t mask)
717 {
718 
719 	__asm __volatile("wrpkru" :  : "a" (mask),  "c" (0), "d" (0));
720 }
721 
722 void    reset_dbregs(void);
723 
724 #ifdef _KERNEL
725 int	rdmsr_safe(u_int msr, uint64_t *val);
726 int	wrmsr_safe(u_int msr, uint64_t newval);
727 #endif
728 
729 #endif /* !_MACHINE_CPUFUNC_H_ */
730