xref: /illumos-gate/usr/src/lib/libsmbfs/smb/crypt.c (revision 4eaa471005973e11a6110b69fe990530b3b95a38)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Crypto support, using libpkcs11
29  *
30  * Some code copied from the server: libsmb smb_crypt.c
31  * with minor changes, i.e. errno.h return values.
32  * XXX: Move this to a common library (later).
33  */
34 
35 #include <sys/types.h>
36 #include <sys/md4.h>
37 
38 #include <errno.h>
39 #include <fcntl.h>
40 #include <string.h>
41 
42 #include <security/cryptoki.h>
43 #include <security/pkcs11.h>
44 #include <cryptoutil.h>
45 
46 #include "smb_crypt.h"
47 
48 static void
49 smb_initlmkey(uchar_t *keyout, const uchar_t *keyin);
50 
51 /*
52  * Like libsmb smb_auth_DES,
53  * but use uchar_t, return errno.
54  */
55 int
56 smb_encrypt_DES(uchar_t *Result, int ResultLen,
57     const uchar_t *Key, int KeyLen,
58     const uchar_t *Data, int DataLen)
59 {
60 	CK_RV rv;
61 	CK_MECHANISM mechanism;
62 	CK_OBJECT_HANDLE hKey;
63 	CK_SESSION_HANDLE hSession;
64 	CK_ULONG ciphertext_len;
65 	uchar_t des_key[8];
66 	int error = 0;
67 	int K, D;
68 	int k, d;
69 
70 	/* Calculate proper number of iterations */
71 	K = KeyLen / 7;
72 	D = DataLen / 8;
73 
74 	if (ResultLen < (K * 8 * D)) {
75 		return (EINVAL);
76 	}
77 
78 	/*
79 	 * Use SUNW convenience function to initialize the cryptoki
80 	 * library, and open a session with a slot that supports
81 	 * the mechanism we plan on using.
82 	 */
83 	mechanism.mechanism = CKM_DES_ECB;
84 	mechanism.pParameter = NULL;
85 	mechanism.ulParameterLen = 0;
86 	rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
87 	if (rv != CKR_OK) {
88 		return (ENOTSUP);
89 	}
90 
91 	for (k = 0; k < K; k++) {
92 		smb_initlmkey(des_key, &Key[k * 7]);
93 		rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
94 		    des_key, 8, &hKey);
95 		if (rv != CKR_OK) {
96 			error = EIO;
97 			goto exit_session;
98 		}
99 		/* Initialize the encryption operation in the session */
100 		rv = C_EncryptInit(hSession, &mechanism, hKey);
101 		if (rv != CKR_OK) {
102 			error = EIO;
103 			goto exit_encrypt;
104 		}
105 		ciphertext_len = DataLen;
106 		for (d = 0; d < D; d++) {
107 			/* Read in the data and encrypt this portion */
108 			rv = C_EncryptUpdate(hSession,
109 			    (CK_BYTE_PTR)Data + (d * 8), 8,
110 			    &Result[(k * (8 * D)) + (d * 8)],
111 			    &ciphertext_len);
112 			if (rv != CKR_OK) {
113 				error = EIO;
114 				goto exit_encrypt;
115 			}
116 		}
117 		(void) C_DestroyObject(hSession, hKey);
118 	}
119 	goto exit_session;
120 
121 exit_encrypt:
122 	(void) C_DestroyObject(hSession, hKey);
123 exit_session:
124 	(void) C_CloseSession(hSession);
125 
126 	return (error);
127 }
128 
129 /*
130  * See "Netlogon Credential Computation" section of MS-NRPC document.
131  * Same as in libsmb, but output arg first.
132  */
133 static void
134 smb_initlmkey(uchar_t *keyout, const uchar_t *keyin)
135 {
136 	int i;
137 
138 	keyout[0] = keyin[0] >> 0x01;
139 	keyout[1] = ((keyin[0] & 0x01) << 6) | (keyin[1] >> 2);
140 	keyout[2] = ((keyin[1] & 0x03) << 5) | (keyin[2] >> 3);
141 	keyout[3] = ((keyin[2] & 0x07) << 4) | (keyin[3] >> 4);
142 	keyout[4] = ((keyin[3] & 0x0f) << 3) | (keyin[4] >> 5);
143 	keyout[5] = ((keyin[4] & 0x1f) << 2) | (keyin[5] >> 6);
144 	keyout[6] = ((keyin[5] & 0x3f) << 1) | (keyin[6] >> 7);
145 	keyout[7] = keyin[6] & 0x7f;
146 
147 	for (i = 0; i < 8; i++)
148 		keyout[i] = (keyout[i] << 1) & 0xfe;
149 }
150 
151 /*
152  * Get some random bytes from /dev/urandom
153  *
154  * There may be a preferred way to call this via libpkcs11
155  * XXX: (see: C_GenerateRandom, etc. -- later...)
156  * Just read from /dev/urandom for now.
157  */
158 int
159 smb_get_urandom(void *data, size_t dlen)
160 {
161 	int fd, rlen;
162 
163 	fd = open("/dev/urandom", O_RDONLY);
164 	if (fd < 0)
165 		return (errno);
166 
167 	rlen = read(fd, data, dlen);
168 	close(fd);
169 
170 	if (rlen < 0)
171 		return (errno);
172 	if (rlen < dlen)
173 		return (EIO);
174 	return (0);
175 }
176