1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, Intel Corporation.
24 * All rights reserved.
25 */
26
27 /*
28 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
29 */
30
31 #ifndef _CRC_HD_H
32 #define _CRC_HD_H
33
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37
38 #include <sys/types.h>
39
40 #ifdef __cplusplus
41 #ifndef INLINE
42 #define INLINE inline
43 #endif
44 #else
45 #ifndef INLINE
46 #define INLINE
47 #endif
48 #endif
49
50 #if defined(__i386) || defined(__amd_64) || defined(__x86_64)
51
52 #ifdef _KERNEL
53 #include <sys/x86_archext.h> /* x86_featureset, X86FSET_SSE4_2 */
54
55 #else
56
57 #include <sys/auxv.h> /* getisax() */
58 #include <sys/auxv_386.h> /* AV_386_SSE4_2 bit */
59
60 #endif
61
62 static INLINE uint32_t
63 #ifdef _KERNEL
64 /*LINTED:E_FUNC_ARG_UNUSED*/
mm_crc32_u8(uint32_t crc,uint8_t * data)65 mm_crc32_u8(uint32_t crc, uint8_t *data)
66 #else
67 mm_crc32_u8(uint32_t crc, uint8_t *data)
68 #endif /* _KERNEL */
69 {
70 __asm__ __volatile__(
71 /* "crc32 r32, r/m8" */
72 ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
73 : "=S" (crc)
74 : "0" (crc), "c" (*data));
75
76 return (crc);
77 }
78
79 #if defined(__amd64) || defined(__x86_64)
80
81 static INLINE uint32_t
82 #ifdef _KERNEL
83 /*LINTED:E_FUNC_ARG_UNUSED*/
mm_crc32_u64(uint32_t crc,uint64_t * data)84 mm_crc32_u64(uint32_t crc, uint64_t *data)
85 #else
86 mm_crc32_u64(uint32_t crc, uint64_t *data)
87 #endif /* _KERNEL */
88 {
89 __asm__ __volatile__(
90 /* "crc32 r32, r/m64" */
91 ".byte 0xF2, 0x48, 0x0F, 0x38, 0xF1, 0xF1"
92 : "=S" (crc)
93 : "0" (crc), "c" (*data));
94
95 return (crc);
96 }
97
98 #define INTEL_CRC_ALIGN_MASK 0x7
99 #define INTEL_CRC_SIZE 8
100 #define INTEL_CRC_DATA_TYPE uint64_t
101 #define INTEL_CRC_FUNC mm_crc32_u64
102
103 #else
104
105 static INLINE uint32_t
106 #ifdef _KERNEL
107 /*LINTED:E_FUNC_ARG_UNUSED*/
mm_crc32_u32(uint32_t crc,uint32_t * data)108 mm_crc32_u32(uint32_t crc, uint32_t *data)
109 #else
110 mm_crc32_u32(uint32_t crc, uint32_t *data)
111 #endif /* _KERNEL */
112 {
113 __asm__ __volatile__(
114 /* "crc32 r32, r/m32" */
115 ".byte 0xF2, 0x0F, 0x38, 0xF1, 0xF1"
116 : "=S" (crc)
117 : "0" (crc), "c" (*data));
118
119 return (crc);
120 }
121
122 #define INTEL_CRC_ALIGN_MASK 0x3
123 #define INTEL_CRC_SIZE 4
124 #define INTEL_CRC_DATA_TYPE uint32_t
125 #define INTEL_CRC_FUNC mm_crc32_u32
126
127 #endif
128
129 static INLINE uint32_t
intel_crc32c(uint8_t * address,unsigned long length,uint32_t crc)130 intel_crc32c(uint8_t *address, unsigned long length, uint32_t crc)
131 {
132 uint32_t i = (uintptr_t)address & INTEL_CRC_ALIGN_MASK;
133 uint8_t *data = address;
134
135 /* Process unaligned header data */
136 while ((length > 0) && (i > 0)) {
137 crc = mm_crc32_u8(crc, data);
138 ++ data;
139 -- length;
140 i ++;
141 i &= INTEL_CRC_ALIGN_MASK;
142 }
143
144 /* aligned data part */
145 while (length >= INTEL_CRC_SIZE) {
146 /*LINTED:E_BAD_PTR_CAST_ALIGN*/
147 crc = INTEL_CRC_FUNC(crc, (INTEL_CRC_DATA_TYPE *)data);
148 data += INTEL_CRC_SIZE;
149 length -= INTEL_CRC_SIZE;
150 }
151
152 /* unaligned tail data */
153 while (length > 0) {
154 crc = mm_crc32_u8(crc, data);
155 ++ data;
156 -- length;
157 }
158
159 return (crc ^ 0xFFFFFFFF);
160 }
161
162 #define HW_CRC32(buffer, length, crc) (intel_crc32c((buffer), (length), (crc)))
163 #define HW_CRC32_CONT(buffer, length, crc) \
164 (intel_crc32c((buffer), (length), (crc) ^ 0xFFFFFFFF))
165 #else
166 #define HW_CRC32(buffer, length, crc) 0
167 #define HW_CRC32_CONT(buffer, length, crc) 0
168 #endif
169
170 static INLINE boolean_t
171 #if defined(_KERNEL) && !defined(__i386) && !defined(__amd_64) &&\
172 !defined(__x86_64)
173 /*LINTED:E_FUNC_ARG_UNUSED*/
hd_crc32_avail(uint32_t * crc32_table)174 hd_crc32_avail(uint32_t *crc32_table)
175 #else
176 hd_crc32_avail(uint32_t *crc32_table)
177 #endif
178 {
179 #if defined(__i386) || defined(__amd_64) || defined(__x86_64)
180 int i;
181 /* poly = 0x1EDC6F41 */
182 static const uint32_t _intel_crc32_hd_table[256] = {
183 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
184 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
185 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
186 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
187 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
188 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
189 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
190 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
191 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
192 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
193 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
194 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
195 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
196 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
197 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
198 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
199 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
200 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
201 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
202 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
203 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
204 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
205 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
206 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
207 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
208 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
209 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
210 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
211 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
212 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
213 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
214 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
215 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
216 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
217 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
218 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
219 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
220 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
221 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
222 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
223 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
224 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
225 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
226 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
227 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
228 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
229 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
230 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
231 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
232 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
233 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
234 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
235 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
236 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
237 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
238 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
239 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
240 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
241 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
242 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
243 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
244 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
245 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
246 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
247 };
248
249 #ifdef _KERNEL
250 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
251 return (B_FALSE);
252 } else {
253 #else
254 {
255 uint_t ui = 0;
256
257 (void) getisax(&ui, 1);
258
259 if (!(ui & AV_386_SSE4_2)) {
260 return (B_FALSE);
261 }
262 #endif /* _KERNEL */
263 for (i = 0; i < 256; i++) {
264 if (crc32_table[i] != _intel_crc32_hd_table[i])
265 return (B_FALSE);
266 }
267 return (B_TRUE);
268 }
269 #else
270 return (B_FALSE);
271 #endif
272 }
273
274 #ifdef __cplusplus
275 }
276 #endif
277
278 #endif /* _CRC_HD_H */
279