1/* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12/* 13 * Copyright 2019 Joyent, Inc. 14 * Copyright 2020 Oxide Computer Company 15 */ 16 17#include <sys/asm_linkage.h> 18#include <sys/segments.h> 19#include <sys/time_impl.h> 20#include <sys/tsc.h> 21#include <cp_offsets.h> 22 23#define GETCPU_GDT_OFFSET SEL_GDT(GDT_CPUID, SEL_UPL) 24 25/* 26 * For __cp_tsc_read calls which incur looping retries due to CPU migration, 27 * this represents the maximum number of tries before bailing out. 28 */ 29#define TSC_READ_MAXLOOP 0x4 30 31 .file "cp_subr.s" 32 33/* 34 * hrtime_t 35 * __cp_tsc_read(comm_page_t *cp) 36 * 37 * Stack usage: 0x18 bytes 38 * 39 * %ebp-0x00 - frame pointer 40 * %ebp-0x04 - saved %edi 41 * %ebp-0x08 - saved %esi 42 * %ebp-0x0c - CPU ID 43 * %ebp-0x10 - loop count 44 * %ebp-0x14 - saved %ebx (during cpuid) 45 */ 46 ENTRY_NP(__cp_tsc_read) 47 pushl %ebp 48 movl %esp, %ebp 49 pushl %edi 50 pushl %esi 51 subl $0x8, %esp 52 53 movl 0x8(%ebp), %edi 54 movl CP_TSC_TYPE(%edi), %eax 55 movl CP_TSC_NCPU(%edi), %esi 56 cmpl $TSC_TSCP, %eax 57 jne 3f 58 rdtscp 59 cmpl $0, %esi 60 jne 2f 611: 62 addl $0x8, %esp 63 popl %esi 64 popl %edi 65 leave 66 ret 672: 68 /* 69 * When cp_tsc_ncpu is non-zero, it indicates the length of the 70 * cp_tsc_sync_tick_delta array, which contains per-CPU offsets for the 71 * TSC. The CPU ID furnished by the IA32_TSC_AUX register via rdtscp 72 * is used to look up an offset value in that array and apply it to the 73 * TSC reading. 74 */ 75 leal CP_TSC_SYNC_TICK_DELTA(%edi), %esi 76 leal (%esi, %ecx, 8), %ecx 77 addl (%ecx), %eax 78 adcl 0x4(%ecx), %edx 79 jmp 1b 80 813: 82 testl %esi, %esi 83 jz 4f 84 85 movl $0, (%esp) 86 mov $GETCPU_GDT_OFFSET, %eax 87 lsl %eax, %eax 88 movl %eax, 0x4(%esp) 89 movl CP_TSC_TYPE(%edi), %eax 90 914: 92 /* 93 * TSC_RDTSC_MFENCE was used in the past for AMD chips, but has been 94 * supplanted by TSC_RDTSC_LFENCE, which works on Intel and AMD (when 95 * lfence can be confirmed as serializing). 96 */ 97 985: 99 cmpl $TSC_RDTSC_LFENCE, %eax 100 jne 6f 101 lfence 102 rdtsc 103 jmp 8f 104 1056: 106 cmpl $TSC_RDTSC_CPUID, %eax 107 jne 7f 108 pushl %ebx 109 xorl %eax, %eax 110 cpuid 111 rdtsc 112 popl %ebx 113 jmp 8f 114 1157: 116 /* 117 * Other protections should have prevented this function from being 118 * called in the first place. Since callers must handle a failure from 119 * CPU migration looping, yield the same result as a bail-out: 0 120 */ 121 xorl %eax, %eax 122 xorl %edx, %edx 123 jmp 1b 124 1258: 126 127 cmpl $0, %esi 128 je 1b 129 /* 130 * With a TSC reading in-hand, confirm that the thread has not migrated 131 * since the cpu_id was first checked. 132 */ 133 movl $GETCPU_GDT_OFFSET, %ecx 134 lsl %ecx, %ecx 135 movl 0x4(%esp), %esi 136 cmpl %ecx, %esi 137 jne 9f 138 139 /* Grab the per-cpu offset and add it to the TSC result */ 140 leal CP_TSC_SYNC_TICK_DELTA(%edi), %esi 141 leal (%esi, %ecx, 8), %ecx 142 addl (%ecx), %eax 143 adcl 0x4(%ecx), %edx 144 jmp 1b 145 1469: 147 /* 148 * It appears that a migration has occurred between the first CPU ID 149 * query and now. Check if the loop limit has been broken and retry if 150 * that's not the case. 151 */ 152 movl (%esp), %eax 153 cmpl $TSC_READ_MAXLOOP, %eax 154 jge 10f 155 156 incl %eax 157 movl %eax, (%esp) 158 movl %ecx, 0x4(%esp) 159 movl CP_TSC_NCPU(%edi), %esi 160 movl CP_TSC_TYPE(%edi), %eax 161 jmp 4b 162 16310: 164 /* Loop limit was reached. Return bail-out value of 0. */ 165 xorl %eax, %eax 166 xorl %edx, %edx 167 jmp 1b 168 169 SET_SIZE(__cp_tsc_read) 170 171/* 172 * hrtime_t 173 * __cp_gethrtime_fasttrap() 174 */ 175 ENTRY_NP(__cp_gethrtime_fasttrap) 176 movl $T_GETHRTIME, %eax 177 int $T_FASTTRAP 178 ret 179 SET_SIZE(__cp_gethrtime_fasttrap) 180 181/* 182 * uint_t 183 * __cp_getcpu(uint_t cp_tsc_type) 184 */ 185 ENTRY_NP(__cp_getcpu) 186 /* 187 * If RDTSCP is available, it is a quick way to grab the cpu_id which 188 * is stored in the TSC_AUX MSR by the kernel. 189 */ 190 movl 4(%esp), %eax 191 movl CP_TSC_TYPE(%eax), %eax 192 cmpl $TSC_TSCP, %eax 193 jne 1f 194 rdtscp 195 movl %ecx, %eax 196 ret 1971: 198 mov $GETCPU_GDT_OFFSET, %eax 199 lsl %eax, %eax 200 ret 201 SET_SIZE(__cp_getcpu) 202