xref: /illumos-gate/usr/src/lib/libsmbfs/smb/crypt.c (revision 4c28a617e3922d92a58e813a5b955eb526b9c386)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
26  */
27 
28 /*
29  * Crypto support, using libpkcs11
30  *
31  * Some code copied from the server: libsmb smb_crypt.c
32  * with minor changes, i.e. errno.h return values.
33  * XXX: Later, make the server use these.
34  */
35 
36 #include <sys/types.h>
37 #include <sys/md4.h>
38 
39 #include <errno.h>
40 #include <fcntl.h>
41 #include <string.h>
42 
43 #include <security/cryptoki.h>
44 #include <security/pkcs11.h>
45 #include <cryptoutil.h>
46 
47 #include "smb_crypt.h"
48 
49 static void
50 smb_initlmkey(uchar_t *keyout, const uchar_t *keyin);
51 
52 /*
53  * Like libsmb smb_auth_DES,
54  * but use uchar_t, return errno.
55  */
56 int
57 smb_encrypt_DES(uchar_t *Result, int ResultLen,
58     const uchar_t *Key, int KeyLen,
59     const uchar_t *Data, int DataLen)
60 {
61 	CK_RV rv;
62 	CK_MECHANISM mechanism;
63 	CK_OBJECT_HANDLE hKey;
64 	CK_SESSION_HANDLE hSession;
65 	CK_ULONG ciphertext_len;
66 	uchar_t des_key[8];
67 	int error = 0;
68 	int K, D;
69 	int k, d;
70 
71 	/*
72 	 * Calculate proper number of iterations.
73 	 * Known call cases include:
74 	 *   ResultLen=16, KeyLen=14, DataLen=8
75 	 *   ResultLen=24, KeyLen=21, DataLen=8
76 	 *   ResultLen=16, KeyLen=14, DataLen=16
77 	 */
78 	K = KeyLen / 7;
79 	D = DataLen / 8;
80 	if ((KeyLen % 7) || (DataLen % 8))
81 		return (EINVAL);
82 	if (K == 0 || D == 0)
83 		return (EINVAL);
84 	if (ResultLen < (K * 8))
85 		return (EINVAL);
86 
87 	/*
88 	 * Use SUNW convenience function to initialize the cryptoki
89 	 * library, and open a session with a slot that supports
90 	 * the mechanism we plan on using.
91 	 */
92 	mechanism.mechanism = CKM_DES_ECB;
93 	mechanism.pParameter = NULL;
94 	mechanism.ulParameterLen = 0;
95 	rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
96 	if (rv != CKR_OK) {
97 		return (ENOTSUP);
98 	}
99 
100 	for (d = k = 0; k < K; k++, d++) {
101 		/* Cycle the input again, as necessary. */
102 		if (d == D)
103 			d = 0;
104 		smb_initlmkey(des_key, &Key[k * 7]);
105 		rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
106 		    des_key, 8, &hKey);
107 		if (rv != CKR_OK) {
108 			error = EIO;
109 			goto exit_session;
110 		}
111 		/* Initialize the encryption operation in the session */
112 		rv = C_EncryptInit(hSession, &mechanism, hKey);
113 		if (rv != CKR_OK) {
114 			error = EIO;
115 			goto exit_encrypt;
116 		}
117 		ciphertext_len = 8;
118 
119 		/* Read in the data and encrypt this portion */
120 		rv = C_EncryptUpdate(hSession,
121 		    (CK_BYTE_PTR)Data + (d * 8), 8,
122 		    (CK_BYTE_PTR)Result + (k * 8),
123 		    &ciphertext_len);
124 		if (rv != CKR_OK) {
125 			error = EIO;
126 			goto exit_encrypt;
127 		}
128 
129 		(void) C_DestroyObject(hSession, hKey);
130 	}
131 	goto exit_session;
132 
133 exit_encrypt:
134 	(void) C_DestroyObject(hSession, hKey);
135 exit_session:
136 	(void) C_CloseSession(hSession);
137 
138 	return (error);
139 }
140 
141 /*
142  * See "Netlogon Credential Computation" section of MS-NRPC document.
143  * Same as in libsmb, but output arg first.
144  */
145 static void
146 smb_initlmkey(uchar_t *keyout, const uchar_t *keyin)
147 {
148 	int i;
149 
150 	keyout[0] = keyin[0] >> 0x01;
151 	keyout[1] = ((keyin[0] & 0x01) << 6) | (keyin[1] >> 2);
152 	keyout[2] = ((keyin[1] & 0x03) << 5) | (keyin[2] >> 3);
153 	keyout[3] = ((keyin[2] & 0x07) << 4) | (keyin[3] >> 4);
154 	keyout[4] = ((keyin[3] & 0x0f) << 3) | (keyin[4] >> 5);
155 	keyout[5] = ((keyin[4] & 0x1f) << 2) | (keyin[5] >> 6);
156 	keyout[6] = ((keyin[5] & 0x3f) << 1) | (keyin[6] >> 7);
157 	keyout[7] = keyin[6] & 0x7f;
158 
159 	for (i = 0; i < 8; i++)
160 		keyout[i] = (keyout[i] << 1) & 0xfe;
161 }
162 
163 /*
164  * CKM_RC4
165  */
166 int
167 smb_encrypt_RC4(uchar_t *Result, int ResultLen,
168 	const uchar_t *Key, int KeyLen,
169 	const uchar_t *Data, int DataLen)
170 {
171 	CK_RV rv;
172 	CK_MECHANISM mechanism;
173 	CK_OBJECT_HANDLE hKey;
174 	CK_SESSION_HANDLE hSession;
175 	CK_ULONG ciphertext_len;
176 	int error = EIO;
177 
178 	/*
179 	 * Use SUNW convenience function to initialize the cryptoki
180 	 * library, and open a session with a slot that supports
181 	 * the mechanism we plan on using.
182 	 */
183 	mechanism.mechanism = CKM_RC4;
184 	mechanism.pParameter = NULL;
185 	mechanism.ulParameterLen = 0;
186 	rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
187 	if (rv != CKR_OK) {
188 		return (ENOTSUP);
189 	}
190 
191 	rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
192 	    Key, KeyLen, &hKey);
193 	if (rv != CKR_OK)
194 		goto exit_session;
195 
196 	/* Initialize the encryption operation in the session */
197 	rv = C_EncryptInit(hSession, &mechanism, hKey);
198 	if (rv != CKR_OK)
199 		goto exit_encrypt;
200 
201 	ciphertext_len = ResultLen;
202 	rv = C_EncryptUpdate(hSession,
203 	    (CK_BYTE_PTR)Data, DataLen,
204 	    (CK_BYTE_PTR)Result, &ciphertext_len);
205 	if (rv == CKR_OK)
206 		error = 0;
207 
208 exit_encrypt:
209 	(void) C_DestroyObject(hSession, hKey);
210 exit_session:
211 	(void) C_CloseSession(hSession);
212 
213 	return (error);
214 }
215 
216 /*
217  * Get some random bytes from /dev/urandom
218  *
219  * There may be a preferred way to call this via libpkcs11
220  * XXX: (see: C_GenerateRandom, etc. -- later...)
221  * Just read from /dev/urandom for now.
222  */
223 int
224 smb_get_urandom(void *data, size_t dlen)
225 {
226 	int fd, rlen;
227 
228 	fd = open("/dev/urandom", O_RDONLY);
229 	if (fd < 0)
230 		return (errno);
231 
232 	rlen = read(fd, data, dlen);
233 	close(fd);
234 
235 	if (rlen < 0)
236 		return (errno);
237 	if (rlen < dlen)
238 		return (EIO);
239 	return (0);
240 }
241