xref: /illumos-gate/usr/src/lib/smbsrv/libsmb/common/smb_auth.c (revision c124a83e09115de88ecccd4f689983f42a1d53bd)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
25  */
26 
27 #include <strings.h>
28 #include <stdlib.h>
29 #include <smbsrv/string.h>
30 #include <smbsrv/libsmb.h>
31 #include <assert.h>
32 
33 /*
34  * smb_auth_qnd_unicode
35  *
36  * Quick and dirty unicode conversion!
37  * Returns the length of dst in bytes.
38  */
39 int
40 smb_auth_qnd_unicode(smb_wchar_t *dst, const char *src, int length)
41 {
42 	int i;
43 	unsigned int count;
44 	smb_wchar_t new_char;
45 
46 	if ((count = oemtoucs(dst, src, length, OEM_CPG_1252)) == 0) {
47 		for (i = 0; i < length; ++i) {
48 			new_char = (smb_wchar_t)src[i] & 0xff;
49 			dst[i] = LE_IN16(&new_char);
50 		}
51 		dst[i] = 0;
52 		count = length;
53 	}
54 
55 	return (count * sizeof (smb_wchar_t));
56 }
57 
58 /*
59  * smb_auth_lmupr
60  *
61  * Converts the given LM password to all uppercase.
62  * The standard strupr cannot
63  * be used here because lm_pwd doesn't have to be
64  * nul terminated.
65  */
66 static void
67 smb_auth_lmupr(unsigned char *lm_pwd)
68 {
69 	unsigned char *p = lm_pwd;
70 	int i;
71 
72 	for (i = 0; (*p) && (i < SMBAUTH_LM_PWD_SZ); i++) {
73 		if (smb_isascii(*p)) {
74 			*p = smb_toupper(*p);
75 			p++;
76 		}
77 	}
78 }
79 
80 /*
81  * smb_auth_lm_hash
82  *
83  * Source: Implementing CIFS (Chris Hertel)
84  *
85  * 1. The password, as entered by user, is either padded with nulls
86  *	  or trimmed to 14 bytes.
87  *    . Note that the 14-byte result string is not handled as a
88  *	    nul-terminated string.
89  *	  . The given password is OEM not Unicode
90  *
91  * 2. The 14-byte password is converted to all uppercase
92  *
93  * 3. The result is used as key to encrypt the KGS magic string to
94  *    make a 16-byte hash.
95  */
96 int
97 smb_auth_lm_hash(const char *password, unsigned char *lm_hash)
98 {
99 	unsigned char lm_pwd[SMBAUTH_LM_PWD_SZ];
100 
101 	bzero((void *)lm_pwd, SMBAUTH_LM_PWD_SZ);
102 	(void) strncpy((char *)lm_pwd, password, SMBAUTH_LM_PWD_SZ);
103 	smb_auth_lmupr(lm_pwd);
104 
105 	return (smb_auth_DES(lm_hash, SMBAUTH_HASH_SZ, lm_pwd,
106 	    SMBAUTH_LM_PWD_SZ, (unsigned char *)SMBAUTH_LM_MAGIC_STR,
107 	    sizeof (SMBAUTH_LM_MAGIC_STR)));
108 }
109 
110 /*
111  * smb_auth_lm_response
112  *
113  * Create a LM response from the given LM hash and challenge.
114  *
115  * Returns SMBAUTH_FAILURE if any problems occur, SMBAUTH_SUCCESS if
116  * all goes well.
117  */
118 static int
119 smb_auth_lm_response(unsigned char *hash,
120     unsigned char *challenge, int clen,
121     unsigned char *lm_rsp)
122 {
123 	unsigned char S21[21];
124 
125 	/*
126 	 * 14-byte LM Hash should be padded with 5 nul bytes to create
127 	 * a 21-byte string to be used in producing LM response
128 	 */
129 	bzero(&S21[SMBAUTH_HASH_SZ], 5);
130 	bcopy(hash, S21, SMBAUTH_HASH_SZ);
131 
132 	/* padded LM Hash -> LM Response */
133 	return (smb_auth_DES(lm_rsp, SMBAUTH_LM_RESP_SZ, S21, 21,
134 	    challenge, clen));
135 }
136 
137 /*
138  * smb_auth_ntlm_hash
139  *
140  * Make NTLM Hash (using MD4) from the given password.
141  * The result will contain a 16-byte NTLM hash.
142  */
143 int
144 smb_auth_ntlm_hash(const char *password, unsigned char *hash)
145 {
146 	smb_wchar_t *unicode_password;
147 	int length, unicode_len;
148 	int rc;
149 
150 	if (password == NULL || hash == NULL)
151 		return (SMBAUTH_FAILURE);
152 
153 	length = strlen(password);
154 	unicode_len = (length + 1) * sizeof (smb_wchar_t);
155 	unicode_password = malloc(unicode_len);
156 
157 	if (unicode_password == NULL)
158 		return (SMBAUTH_FAILURE);
159 
160 	length = smb_auth_qnd_unicode(unicode_password, password, length);
161 	rc = smb_auth_md4(hash, (unsigned char *)unicode_password, length);
162 
163 	(void) memset(unicode_password, 0, unicode_len);
164 	free(unicode_password);
165 
166 	return (rc);
167 }
168 
169 /*
170  * smb_auth_ntlm_response
171  *
172  * Make LM/NTLM response from the given LM/NTLM Hash and given
173  * challenge.
174  */
175 static int
176 smb_auth_ntlm_response(unsigned char *hash,
177     unsigned char *challenge, int clen,
178     unsigned char *ntlm_rsp)
179 {
180 	unsigned char S21[21];
181 
182 	bcopy(hash, S21, SMBAUTH_HASH_SZ);
183 	bzero(&S21[SMBAUTH_HASH_SZ], 5);
184 	if (smb_auth_DES((unsigned char *)ntlm_rsp, SMBAUTH_LM_RESP_SZ,
185 	    S21, 21, challenge, clen) == SMBAUTH_FAILURE)
186 		return (0);
187 	return (SMBAUTH_LM_RESP_SZ);
188 }
189 
190 /*
191  * smb_auth_ntlmv2_hash
192  *
193  * The NTLM v2 hash will be created from the given NTLM hash, username,
194  * and the NETBIOS name of the domain.
195  *
196  * The NTLMv2 hash will be returned via the ntlmv2_hash parameter which
197  * will be used in the calculation of the NTLMv2 and LMv2 responses.
198  */
199 int
200 smb_auth_ntlmv2_hash(unsigned char *ntlm_hash,
201     char *username,
202     char *ntdomain,
203     unsigned char *ntlmv2_hash)
204 {
205 	smb_wchar_t *data;
206 	int data_len;
207 	unsigned char *buf;
208 	int rc;
209 
210 	if (username == NULL || ntdomain == NULL)
211 		return (SMBAUTH_FAILURE);
212 
213 	(void) smb_strupr(username);
214 
215 	data_len = strlen(username) + strlen(ntdomain);
216 	buf = (unsigned char *)malloc((data_len + 1) * sizeof (char));
217 	if (buf == NULL)
218 		return (SMBAUTH_FAILURE);
219 
220 	(void) snprintf((char *)buf, data_len + 1, "%s%s", username, ntdomain);
221 	data = (smb_wchar_t *)malloc((data_len + 1) * sizeof (smb_wchar_t));
222 	if (data == NULL) {
223 		free(buf);
224 		return (SMBAUTH_FAILURE);
225 	}
226 
227 	data_len = smb_auth_qnd_unicode(data, (char *)buf, data_len);
228 	rc = SMBAUTH_HMACT64((unsigned char *)data, data_len, ntlm_hash,
229 	    SMBAUTH_HASH_SZ, ntlmv2_hash);
230 
231 	free(buf);
232 	free(data);
233 	return (rc);
234 }
235 
236 /*
237  * smb_auth_v2_response
238  *
239  * Caculates either the LMv2 or NTLMv2 response.
240  *
241  * Same algorithm is used for calculating both LMv2 or NTLMv2 responses.
242  * This routine will return NTLMv2 response if the data blob information
243  * is passed in as the clnt_data. Otherwise, it will return LMv2 response
244  * with the 8-byte client challenge(a.k.a blip) as the clnt_data.
245  *
246  * (LM/NTLM)v2 response is the hmac-md5 hash of the specified data
247  * (server challenge + NTLMv2 data blob or LMv2 client challenge)
248  * using the NTLMv2 hash as the key.
249  *
250  * Returns the size of the corresponding v2 response upon success.
251  * Otherwise, returns -1 on error.
252  */
253 static int
254 smb_auth_v2_response(
255 	unsigned char *hash,
256 	unsigned char *srv_challenge, int slen,
257 	unsigned char *clnt_data, int clen,
258 	unsigned char *v2_rsp)
259 {
260 	unsigned char *hmac_data;
261 
262 	hmac_data = (unsigned char *)malloc((slen + clen) * sizeof (char));
263 	if (!hmac_data) {
264 		return (-1);
265 	}
266 
267 	(void) memcpy(hmac_data, srv_challenge, slen);
268 	(void) memcpy(&hmac_data[slen], clnt_data, clen);
269 	if (SMBAUTH_HMACT64(hmac_data, slen + clen, (unsigned char *)hash,
270 	    SMBAUTH_HASH_SZ, (unsigned char *)v2_rsp) != SMBAUTH_SUCCESS)
271 		return (-1);
272 	(void) memcpy(&v2_rsp[SMBAUTH_HASH_SZ], clnt_data, clen);
273 
274 	free(hmac_data);
275 	return (SMBAUTH_HASH_SZ + clen);
276 }
277 
278 /*
279  * smb_auth_gen_session_key
280  *
281  * Generate the NTLM user session key if LMCompatibilityLevel is 2 or
282  * NTLMv2 user session key if LMCompatibilityLevel is 3 or above.
283  *
284  * NTLM_Session_Key = MD4(NTLM_Hash);
285  *
286  * NTLMv2_Session_Key = HMAC_MD5(NTLMv2Hash, 16, NTLMv2_HMAC, 16)
287  *
288  * Prior to calling this function, the auth instance should be set
289  * via smb_auth_set_info().
290  *
291  * Returns the appropriate session key.
292  */
293 int
294 smb_auth_gen_session_key(smb_auth_info_t *auth, unsigned char *session_key)
295 {
296 	int rc;
297 
298 	if (auth->lmcompatibility_lvl == 2)
299 		rc = smb_auth_md4(session_key, auth->hash, SMBAUTH_HASH_SZ);
300 	else
301 		rc = SMBAUTH_HMACT64((unsigned char *)auth->cs,
302 		    SMBAUTH_HASH_SZ, (unsigned char *)auth->hash_v2,
303 		    SMBAUTH_SESSION_KEY_SZ, session_key);
304 
305 	return (rc);
306 }
307 
308 static boolean_t
309 smb_lm_password_ok(
310     unsigned char *challenge,
311     uint32_t clen,
312     unsigned char *lm_hash,
313     unsigned char *passwd)
314 {
315 	unsigned char lm_resp[SMBAUTH_LM_RESP_SZ];
316 	int rc;
317 
318 	rc = smb_auth_lm_response(lm_hash, challenge, clen, lm_resp);
319 	if (rc != SMBAUTH_SUCCESS)
320 		return (B_FALSE);
321 
322 	return (bcmp(lm_resp, passwd, SMBAUTH_LM_RESP_SZ) == 0);
323 }
324 
325 static boolean_t
326 smb_ntlm_password_ok(
327     unsigned char *challenge,
328     uint32_t clen,
329     unsigned char *ntlm_hash,
330     unsigned char *passwd,
331     unsigned char *session_key)
332 {
333 	unsigned char ntlm_resp[SMBAUTH_LM_RESP_SZ];
334 	int rc;
335 	boolean_t ok;
336 
337 	rc = smb_auth_ntlm_response(ntlm_hash, challenge, clen, ntlm_resp);
338 	if (rc != SMBAUTH_LM_RESP_SZ)
339 		return (B_FALSE);
340 
341 	ok = (bcmp(ntlm_resp, passwd, SMBAUTH_LM_RESP_SZ) == 0);
342 	if (ok && (session_key)) {
343 		rc = smb_auth_md4(session_key, ntlm_hash, SMBAUTH_HASH_SZ);
344 		if (rc != SMBAUTH_SUCCESS)
345 			ok = B_FALSE;
346 	}
347 	return (ok);
348 }
349 
350 static boolean_t
351 smb_ntlmv2_password_ok(
352     unsigned char *challenge,
353     uint32_t clen,
354     unsigned char *ntlm_hash,
355     unsigned char *passwd,
356     int pwdlen,
357     char *domain,
358     char *username,
359     uchar_t *session_key)
360 {
361 	unsigned char *clnt_blob;
362 	int clnt_blob_len;
363 	unsigned char ntlmv2_hash[SMBAUTH_HASH_SZ];
364 	unsigned char *ntlmv2_resp;
365 	boolean_t ok = B_FALSE;
366 	char *dest[3];
367 	int i;
368 	int rc;
369 
370 	clnt_blob_len = pwdlen - SMBAUTH_HASH_SZ;
371 	clnt_blob = &passwd[SMBAUTH_HASH_SZ];
372 	dest[0] = domain;
373 	if ((dest[1] = strdup(domain)) == NULL)
374 		return (B_FALSE);
375 	(void) smb_strupr(dest[1]);
376 	dest[2] = "";
377 
378 	/*
379 	 * 15.5.2 The NTLMv2 Password Hash, pg. 279, of the "Implementing CIFS"
380 	 *
381 	 * The NTLMv2 Hash is created from:
382 	 * - NTLM hash
383 	 * - user's username, and
384 	 * - the name of the logon destination(i.e. the NetBIOS name of either
385 	 *   the SMB server or NT Domain against which the user is trying to
386 	 *   authenticate.
387 	 *
388 	 * Experiments show this is not exactly the case.
389 	 * For Windows Server 2003, the domain name needs to be included and
390 	 * converted to uppercase. For Vista, the domain name needs to be
391 	 * included also, but leave the case alone.  And in some cases it needs
392 	 * to be empty. All three variants are tried here.
393 	 */
394 
395 	ntlmv2_resp = (unsigned char *)malloc(SMBAUTH_HASH_SZ + clnt_blob_len);
396 	if (ntlmv2_resp == NULL) {
397 		free(dest[1]);
398 		return (B_FALSE);
399 	}
400 
401 	for (i = 0; i < (sizeof (dest) / sizeof (char *)); i++) {
402 		if (smb_auth_ntlmv2_hash(ntlm_hash, username, dest[i],
403 		    ntlmv2_hash) != SMBAUTH_SUCCESS)
404 			break;
405 
406 		if (smb_auth_v2_response(ntlmv2_hash, challenge,
407 		    clen, clnt_blob, clnt_blob_len, ntlmv2_resp) < 0)
408 			break;
409 
410 		ok = (bcmp(passwd, ntlmv2_resp, pwdlen) == 0);
411 		if (ok && session_key) {
412 			rc = SMBAUTH_HMACT64(ntlmv2_resp,
413 			    SMBAUTH_HASH_SZ, ntlmv2_hash,
414 			    SMBAUTH_SESSION_KEY_SZ, session_key);
415 			if (rc != SMBAUTH_SUCCESS) {
416 				ok = B_FALSE;
417 			}
418 			break;
419 		}
420 	}
421 
422 	free(dest[1]);
423 	free(ntlmv2_resp);
424 	return (ok);
425 }
426 
427 static boolean_t
428 smb_lmv2_password_ok(
429     unsigned char *challenge,
430     uint32_t clen,
431     unsigned char *ntlm_hash,
432     unsigned char *passwd,
433     char *domain,
434     char *username)
435 {
436 	unsigned char *clnt_challenge;
437 	unsigned char ntlmv2_hash[SMBAUTH_HASH_SZ];
438 	unsigned char lmv2_resp[SMBAUTH_LM_RESP_SZ];
439 	boolean_t ok = B_FALSE;
440 	char *dest[3];
441 	int i;
442 
443 	clnt_challenge = &passwd[SMBAUTH_HASH_SZ];
444 	dest[0] = domain;
445 	if ((dest[1] = strdup(domain)) == NULL)
446 		return (B_FALSE);
447 	(void) smb_strupr(dest[1]);
448 	dest[2] = "";
449 
450 	/*
451 	 * 15.5.2 The NTLMv2 Password Hash, pg. 279, of the "Implementing CIFS"
452 	 *
453 	 * The NTLMv2 Hash is created from:
454 	 * - NTLM hash
455 	 * - user's username, and
456 	 * - the name of the logon destination(i.e. the NetBIOS name of either
457 	 *   the SMB server or NT Domain against which the suer is trying to
458 	 *   authenticate.
459 	 *
460 	 * Experiments show this is not exactly the case.
461 	 * For Windows Server 2003, the domain name needs to be included and
462 	 * converted to uppercase. For Vista, the domain name needs to be
463 	 * included also, but leave the case alone.  And in some cases it needs
464 	 * to be empty. All three variants are tried here.
465 	 */
466 
467 	for (i = 0; i < (sizeof (dest) / sizeof (char *)); i++) {
468 		if (smb_auth_ntlmv2_hash(ntlm_hash, username, dest[i],
469 		    ntlmv2_hash) != SMBAUTH_SUCCESS)
470 			break;
471 
472 		if (smb_auth_v2_response(ntlmv2_hash, challenge,
473 		    clen, clnt_challenge, SMBAUTH_V2_CLNT_CHALLENGE_SZ,
474 		    lmv2_resp) < 0)
475 			break;
476 
477 		ok = (bcmp(passwd, lmv2_resp, SMBAUTH_LM_RESP_SZ) == 0);
478 		if (ok)
479 			break;
480 	}
481 
482 	free(dest[1]);
483 	return (ok);
484 }
485 
486 /*
487  * smb_auth_validate_lm
488  *
489  * Validates given LM/LMv2 client response, passed in passwd arg, against
490  * stored user's password, passed in smbpw
491  *
492  * If LM level <=3 server accepts LM responses, otherwise LMv2
493  */
494 boolean_t
495 smb_auth_validate_lm(
496     unsigned char *challenge,
497     uint32_t clen,
498     smb_passwd_t *smbpw,
499     unsigned char *passwd,
500     int pwdlen,
501     char *domain,
502     char *username)
503 {
504 	boolean_t ok = B_FALSE;
505 	int64_t lmlevel;
506 
507 	if (pwdlen != SMBAUTH_LM_RESP_SZ)
508 		return (B_FALSE);
509 
510 	if (smb_config_getnum(SMB_CI_LM_LEVEL, &lmlevel) != SMBD_SMF_OK)
511 		return (B_FALSE);
512 
513 	if (lmlevel <= 3) {
514 		ok = smb_lm_password_ok(challenge, clen, smbpw->pw_lmhash,
515 		    passwd);
516 	}
517 
518 	if (!ok)
519 		ok = smb_lmv2_password_ok(challenge, clen, smbpw->pw_nthash,
520 		    passwd, domain, username);
521 
522 	return (ok);
523 }
524 
525 /*
526  * smb_auth_validate_nt
527  *
528  * Validates given NTLM/NTLMv2 client response, passed in passwd arg, against
529  * stored user's password, passed in smbpw
530  *
531  * If LM level <=4 server accepts NTLM/NTLMv2 responses, otherwise only NTLMv2
532  */
533 boolean_t
534 smb_auth_validate_nt(
535     unsigned char *challenge,
536     uint32_t clen,
537     smb_passwd_t *smbpw,
538     unsigned char *passwd,
539     int pwdlen,
540     char *domain,
541     char *username,
542     uchar_t *session_key)
543 {
544 	int64_t lmlevel;
545 	boolean_t ok;
546 
547 	if (smb_config_getnum(SMB_CI_LM_LEVEL, &lmlevel) != SMBD_SMF_OK)
548 		return (B_FALSE);
549 
550 	if ((lmlevel == 5) && (pwdlen <= SMBAUTH_LM_RESP_SZ))
551 		return (B_FALSE);
552 
553 	if (pwdlen > SMBAUTH_LM_RESP_SZ)
554 		ok = smb_ntlmv2_password_ok(challenge, clen,
555 		    smbpw->pw_nthash, passwd, pwdlen,
556 		    domain, username, session_key);
557 	else
558 		ok = smb_ntlm_password_ok(challenge, clen,
559 		    smbpw->pw_nthash, passwd, session_key);
560 
561 	return (ok);
562 }
563 
564 /*
565  * smb_gen_random_passwd(buf, len)
566  * Generate a random password of length len-1, and store it in buf,
567  * null terminated.  This is used as a machine account password,
568  * which we set when we join a domain.
569  *
570  * [MS-DISO] A machine password is an ASCII string of randomly chosen
571  * characters. Each character's ASCII code is between 32 and 122 inclusive.
572  * That's space through 'z'.
573  */
574 
575 int
576 smb_gen_random_passwd(char *buf, size_t len)
577 {
578 	const uchar_t start = ' ';
579 	const uchar_t modulus = 'z' - ' ' + 1;
580 	uchar_t t;
581 	int i;
582 
583 	/* Last byte is the null. */
584 	len--;
585 
586 	/* Temporarily put random data in the caller's buffer. */
587 	randomize(buf, len);
588 
589 	/* Convert the random data to printable characters. */
590 	for (i = 0; i < len; i++) {
591 		/* need unsigned math */
592 		t = (uchar_t)buf[i];
593 		t = (t % modulus) + start;
594 		assert(' ' <= t && t <= 'z');
595 		buf[i] = (char)t;
596 	}
597 
598 	buf[len] = '\0';
599 
600 	return (0);
601 }
602