xref: /illumos-gate/usr/src/lib/smbsrv/libfksmbsrv/common/fksmb_preauth_pkcs.c (revision dd72704bd9e794056c558153663c739e2012d721)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2021 RackTop Systems, Inc.
14  */
15 
16 #include <stdlib.h>
17 #include <smbsrv/smb_kproto.h>
18 #include <smbsrv/smb_kcrypt.h>
19 #include <security/cryptoki.h>
20 #include <security/pkcs11.h>
21 
22 /*
23  * SMB 3.1.1 Preauth Integrity
24  */
25 static int
26 getmech_sha512(smb_crypto_mech_t *mech)
27 {
28 	ulong_t mid = CKM_SHA512;
29 	CK_SESSION_HANDLE hdl;
30 	CK_RV rv;
31 
32 	rv = SUNW_C_GetMechSession(mid, &hdl);
33 	if (rv != CKR_OK) {
34 		cmn_err(CE_NOTE, "PKCS#11: no mech 0x%x",
35 		    (unsigned int)mid);
36 		return (-1);
37 	}
38 	(void) C_CloseSession(hdl);
39 
40 	mech->mechanism = mid;
41 	mech->pParameter = NULL;
42 	mech->ulParameterLen = 0;
43 	return (0);
44 }
45 
46 /*
47  * (called from smb2_negotiate_common)
48  */
49 void
50 smb31_preauth_init_mech(smb_session_t *s)
51 {
52 	smb_crypto_mech_t *mech;
53 	int rc;
54 
55 	ASSERT3S(s->dialect, >=, SMB_VERS_3_11);
56 
57 	if (s->preauth_mech != NULL)
58 		return;
59 
60 	mech = kmem_zalloc(sizeof (*mech), KM_SLEEP);
61 	rc = getmech_sha512(mech);
62 	if (rc != 0) {
63 		kmem_free(mech, sizeof (*mech));
64 		return;
65 	}
66 	s->preauth_mech = mech;
67 }
68 
69 void
70 smb31_preauth_fini(smb_session_t *s)
71 {
72 	smb_crypto_mech_t *mech;
73 
74 	if ((mech = s->preauth_mech) != NULL) {
75 		kmem_free(mech, sizeof (*mech));
76 		s->preauth_mech = NULL;
77 	}
78 }
79 
80 /*
81  * Start the KCF session, load the key
82  */
83 int
84 smb_sha512_init(smb_sign_ctx_t *ctxp, smb_crypto_mech_t *mech)
85 {
86 	CK_RV rv;
87 
88 	rv = SUNW_C_GetMechSession(mech->mechanism, ctxp);
89 	if (rv != CKR_OK)
90 		return (-1);
91 
92 	rv = C_DigestInit(*ctxp, mech);
93 
94 	return (rv == CKR_OK ? 0 : -1);
95 }
96 
97 /*
98  * Digest one segment
99  */
100 int
101 smb_sha512_update(smb_sign_ctx_t ctx, void *buf, size_t len)
102 {
103 	CK_RV rv;
104 
105 	rv = C_DigestUpdate(ctx, buf, len);
106 	if (rv != CKR_OK)
107 		(void) C_CloseSession(ctx);
108 
109 	return (rv == CKR_OK ? 0 : -1);
110 }
111 
112 /*
113  * Get the final digest.
114  */
115 int
116 smb_sha512_final(smb_sign_ctx_t ctx, uint8_t *digest)
117 {
118 	CK_ULONG len = SHA512_DIGEST_LENGTH;
119 	CK_RV rv;
120 
121 	rv = C_DigestFinal(ctx, digest, &len);
122 	(void) C_CloseSession(ctx);
123 
124 	return (rv == CKR_OK ? 0 : -1);
125 }
126 
127 int
128 smb31_preauth_sha512_calc(smb_request_t *sr, struct mbuf_chain *mbc,
129     uint8_t *in_hashval, uint8_t *out_hashval)
130 {
131 	smb_session_t *s = sr->session;
132 	smb_sign_ctx_t ctx = 0;
133 	struct mbuf *mbuf = mbc->chain;
134 	int rc;
135 
136 	ASSERT3U(s->smb31_preauth_hashid, !=, 0);
137 
138 	if (s->preauth_mech == NULL)
139 		return (-1);
140 
141 	if ((rc = smb_sha512_init(&ctx, s->preauth_mech)) != 0)
142 		return (rc);
143 
144 	/* Digest current hashval */
145 	rc = smb_sha512_update(ctx, in_hashval, SHA512_DIGEST_LENGTH);
146 	if (rc != 0)
147 		return (rc);
148 
149 	while (mbuf != NULL) {
150 		rc = smb_sha512_update(ctx, mbuf->m_data, mbuf->m_len);
151 		if (rc != 0)
152 			return (rc);
153 		mbuf = mbuf->m_next;
154 	}
155 
156 	rc = smb_sha512_final(ctx, out_hashval);
157 	return (rc);
158 }
159