xref: /illumos-gate/usr/src/lib/smbsrv/libfksmbsrv/common/fksmb_sign_pkcs.c (revision a4568e19224dbd8e405999c57ff8d4e4fd0d877f)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
14  * Copyright 2022 RackTop Systems, Inc.
15  */
16 
17 /*
18  * Helper functions for SMB signing using PKCS#11
19  *
20  * There are two implementations of these functions:
21  * This one (for user space) and another for kernel.
22  * See: uts/common/fs/smbsrv/smb_sign_kcf.c
23  */
24 
25 #include <stdlib.h>
26 #include <smbsrv/smb_kproto.h>
27 #include <smbsrv/smb_kcrypt.h>
28 #include <security/cryptoki.h>
29 #include <security/pkcs11.h>
30 
31 /*
32  * Common function to see if a mech is available.
33  */
34 static int
find_mech(smb_crypto_mech_t * mech,ulong_t mid)35 find_mech(smb_crypto_mech_t *mech, ulong_t mid)
36 {
37 	CK_SESSION_HANDLE hdl;
38 	CK_RV rv;
39 
40 	rv = SUNW_C_GetMechSession(mid, &hdl);
41 	if (rv != CKR_OK) {
42 		cmn_err(CE_NOTE, "PKCS#11: no mech 0x%x",
43 		    (unsigned int)mid);
44 		return (-1);
45 	}
46 	(void) C_CloseSession(hdl);
47 
48 	mech->mechanism = mid;
49 	mech->pParameter = NULL;
50 	mech->ulParameterLen = 0;
51 	return (0);
52 }
53 
54 /*
55  * SMB1 signing helpers:
56  * (getmech, init, update, final)
57  */
58 
59 /*
60  * Find out if we have this mech.
61  */
62 int
smb_md5_getmech(smb_crypto_mech_t * mech)63 smb_md5_getmech(smb_crypto_mech_t *mech)
64 {
65 	return (find_mech(mech, CKM_MD5));
66 }
67 
68 /*
69  * Start PKCS#11 session.
70  */
71 int
smb_md5_init(smb_sign_ctx_t * ctxp,smb_crypto_mech_t * mech)72 smb_md5_init(smb_sign_ctx_t *ctxp, smb_crypto_mech_t *mech)
73 {
74 	CK_RV rv;
75 
76 	rv = SUNW_C_GetMechSession(mech->mechanism, ctxp);
77 	if (rv != CKR_OK)
78 		return (-1);
79 
80 	rv = C_DigestInit(*ctxp, mech);
81 
82 	return (rv == CKR_OK ? 0 : -1);
83 }
84 
85 /*
86  * Digest one segment
87  */
88 int
smb_md5_update(smb_sign_ctx_t ctx,void * buf,size_t len)89 smb_md5_update(smb_sign_ctx_t ctx, void *buf, size_t len)
90 {
91 	CK_RV rv;
92 
93 	rv = C_DigestUpdate(ctx, buf, len);
94 	if (rv != CKR_OK)
95 		(void) C_CloseSession(ctx);
96 
97 	return (rv == CKR_OK ? 0 : -1);
98 }
99 
100 /*
101  * Get the final digest.
102  */
103 int
smb_md5_final(smb_sign_ctx_t ctx,uint8_t * digest16)104 smb_md5_final(smb_sign_ctx_t ctx, uint8_t *digest16)
105 {
106 	CK_ULONG len = MD5_DIGEST_LENGTH;
107 	CK_RV rv;
108 
109 	rv = C_DigestFinal(ctx, digest16, &len);
110 	(void) C_CloseSession(ctx);
111 
112 	return (rv == CKR_OK ? 0 : -1);
113 }
114 
115 /*
116  * SMB2 signing helpers:
117  * (getmech, init, update, final)
118  */
119 
120 /*
121  * Find out if we have this mech.
122  */
123 int
smb2_hmac_getmech(smb_crypto_mech_t * mech)124 smb2_hmac_getmech(smb_crypto_mech_t *mech)
125 {
126 	return (find_mech(mech, CKM_SHA256_HMAC));
127 }
128 
129 /*
130  * Start PKCS#11 session, load the key.
131  */
132 int
smb2_hmac_init(smb_sign_ctx_t * ctxp,smb_crypto_mech_t * mech,uint8_t * key,size_t key_len)133 smb2_hmac_init(smb_sign_ctx_t *ctxp, smb_crypto_mech_t *mech,
134     uint8_t *key, size_t key_len)
135 {
136 	CK_OBJECT_HANDLE hkey = 0;
137 	CK_RV rv;
138 
139 	rv = SUNW_C_GetMechSession(mech->mechanism, ctxp);
140 	if (rv != CKR_OK)
141 		return (-1);
142 
143 	rv = SUNW_C_KeyToObject(*ctxp, mech->mechanism,
144 	    key, key_len, &hkey);
145 	if (rv != CKR_OK)
146 		return (-1);
147 
148 	rv = C_SignInit(*ctxp, mech, hkey);
149 	(void) C_DestroyObject(*ctxp, hkey);
150 
151 	return (rv == CKR_OK ? 0 : -1);
152 }
153 
154 /*
155  * Digest one segment
156  */
157 int
smb2_hmac_update(smb_sign_ctx_t ctx,uint8_t * in,size_t len)158 smb2_hmac_update(smb_sign_ctx_t ctx, uint8_t *in, size_t len)
159 {
160 	CK_RV rv;
161 
162 	rv = C_SignUpdate(ctx, in, len);
163 	if (rv != CKR_OK)
164 		(void) C_CloseSession(ctx);
165 
166 	return (rv == CKR_OK ? 0 : -1);
167 }
168 
169 /*
170  * Note, the SMB2 signature is the first 16 bytes of the
171  * 32-byte SHA256 HMAC digest.  This is specifically for
172  * SMB2 signing, and NOT a generic HMAC function.
173  */
174 int
smb2_hmac_final(smb_sign_ctx_t ctx,uint8_t * digest16)175 smb2_hmac_final(smb_sign_ctx_t ctx, uint8_t *digest16)
176 {
177 	uint8_t full_digest[SHA256_DIGEST_LENGTH];
178 	CK_ULONG len = SHA256_DIGEST_LENGTH;
179 	CK_RV rv;
180 
181 	rv = C_SignFinal(ctx, full_digest, &len);
182 	if (rv == CKR_OK)
183 		bcopy(full_digest, digest16, 16);
184 
185 	(void) C_CloseSession(ctx);
186 
187 	return (rv == CKR_OK ? 0 : -1);
188 }
189 
190 /*
191  * One-shot HMAC function used in smb3_kdf
192  */
193 int
smb2_hmac_one(smb_crypto_mech_t * mech,uint8_t * key,size_t key_len,uint8_t * data,size_t data_len,uint8_t * mac,size_t mac_len)194 smb2_hmac_one(smb_crypto_mech_t *mech,
195     uint8_t *key, size_t key_len,
196     uint8_t *data, size_t data_len,
197     uint8_t *mac, size_t mac_len)
198 {
199 	CK_SESSION_HANDLE hssn = 0;
200 	CK_OBJECT_HANDLE hkey = 0;
201 	CK_ULONG ck_maclen = mac_len;
202 	CK_RV rv;
203 	int rc = 0;
204 
205 	rv = SUNW_C_GetMechSession(mech->mechanism, &hssn);
206 	if (rv != CKR_OK)
207 		return (-1);
208 
209 	rv = SUNW_C_KeyToObject(hssn, mech->mechanism,
210 	    key, key_len, &hkey);
211 	if (rv != CKR_OK) {
212 		rc = -2;
213 		goto out;
214 	}
215 
216 	rv = C_SignInit(hssn, mech, hkey);
217 	if (rv != CKR_OK) {
218 		rc = -3;
219 		goto out;
220 	}
221 
222 	rv = C_Sign(hssn, data, data_len, mac, &ck_maclen);
223 	if (rv != CKR_OK) {
224 		rc = -4;
225 		goto out;
226 	}
227 
228 	if (ck_maclen != mac_len) {
229 		rc = -5;
230 		goto out;
231 	}
232 	rc = 0;
233 
234 out:
235 	if (hkey != 0)
236 		(void) C_DestroyObject(hssn, hkey);
237 	if (hssn != 0)
238 		(void) C_CloseSession(hssn);
239 
240 	return (rc);
241 }
242 
243 /*
244  * SMB3 signing helpers:
245  * (getmech, init, update, final)
246  */
247 
248 /*
249  * Find out if we have this mech.
250  */
251 int
smb3_cmac_getmech(smb_crypto_mech_t * mech)252 smb3_cmac_getmech(smb_crypto_mech_t *mech)
253 {
254 	return (find_mech(mech, CKM_AES_CMAC));
255 }
256 
257 /*
258  * Start PKCS#11 session, load the key.
259  */
260 int
smb3_cmac_init(smb_sign_ctx_t * ctxp,smb_crypto_mech_t * mech,uint8_t * key,size_t key_len)261 smb3_cmac_init(smb_sign_ctx_t *ctxp, smb_crypto_mech_t *mech,
262     uint8_t *key, size_t key_len)
263 {
264 	CK_OBJECT_HANDLE hkey = 0;
265 	CK_RV rv;
266 
267 	rv = SUNW_C_GetMechSession(mech->mechanism, ctxp);
268 	if (rv != CKR_OK)
269 		return (-1);
270 
271 	rv = SUNW_C_KeyToObject(*ctxp, mech->mechanism,
272 	    key, key_len, &hkey);
273 	if (rv != CKR_OK) {
274 		(void) C_CloseSession(*ctxp);
275 		return (-1);
276 	}
277 
278 	rv = C_SignInit(*ctxp, mech, hkey);
279 	(void) C_DestroyObject(*ctxp, hkey);
280 	if (rv != CKR_OK) {
281 		(void) C_CloseSession(*ctxp);
282 		return (-1);
283 	}
284 
285 	return (0);
286 }
287 
288 /*
289  * Digest one segment
290  */
291 int
smb3_cmac_update(smb_sign_ctx_t ctx,uint8_t * in,size_t len)292 smb3_cmac_update(smb_sign_ctx_t ctx, uint8_t *in, size_t len)
293 {
294 	CK_RV rv;
295 
296 	rv = C_SignUpdate(ctx, in, len);
297 	if (rv != CKR_OK)
298 		(void) C_CloseSession(ctx);
299 
300 	return (rv == CKR_OK ? 0 : -1);
301 }
302 
303 /*
304  * Note, the SMB2 signature is just the AES CMAC digest.
305  * (both are 16 bytes long)
306  */
307 int
smb3_cmac_final(smb_sign_ctx_t ctx,uint8_t * digest)308 smb3_cmac_final(smb_sign_ctx_t ctx, uint8_t *digest)
309 {
310 	CK_ULONG len = SMB2_SIG_SIZE;
311 	CK_RV rv;
312 
313 	rv = C_SignFinal(ctx, digest, &len);
314 	(void) C_CloseSession(ctx);
315 
316 	return (rv == CKR_OK ? 0 : -1);
317 }
318