xref: /illumos-gate/usr/src/lib/rpcsec_gss/rpcsec_gss_misc.c (revision 7a6d80f1660abd4755c68cbd094d4a914681d26e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright (c) 1986-1995, 1997, 2001 by Sun Microsystems, Inc.
24  * All rights reserved.
25  */
26 
27 /*
28  * Copyright 1993 OpenVision Technologies, Inc., All Rights Reserved.
29  *
30  * $Header:
31  * /afs/gza.com/product/secure/rel-eng/src/1.1/rpc/RCS/auth_gssapi_misc.c,v
32  * 1.10 1994/10/27 12:39:23 jik Exp $
33  */
34 
35 #include <stdlib.h>
36 #include <gssapi/gssapi.h>
37 #include <rpc/rpc.h>
38 #include <rpc/rpcsec_defs.h>
39 
40 /*
41  * Miscellaneous XDR routines.
42  */
43 bool_t
44 __xdr_gss_buf(xdrs, buf)
45 	XDR		*xdrs;
46 	gss_buffer_t	buf;
47 {
48 	u_int cast_len, bound_len;
49 
50 	/*
51 	 * We go through this contortion because size_t is a now a ulong,
52 	 * GSS-API uses ulongs.
53 	 */
54 
55 	if (xdrs->x_op != XDR_DECODE) {
56 		bound_len = cast_len = (u_int) buf->length;
57 	} else {
58 		bound_len = (u_int)-1;
59 	}
60 
61 	if (xdr_bytes(xdrs, (char **)&buf->value, &cast_len,
62 	    bound_len) == TRUE) {
63 		if (xdrs->x_op == XDR_DECODE)
64 			buf->length = cast_len;
65 
66 		return (TRUE);
67 	}
68 
69 	return (FALSE);
70 }
71 
72 bool_t
73 __xdr_rpc_gss_creds(xdrs, creds)
74 	XDR			*xdrs;
75 	rpc_gss_creds		*creds;
76 {
77 	if (!xdr_u_int(xdrs, &creds->version) ||
78 				!xdr_u_int(xdrs, &creds->gss_proc) ||
79 				!xdr_u_int(xdrs, &creds->seq_num) ||
80 				!xdr_u_int(xdrs, (u_int *)&creds->service) ||
81 				!__xdr_gss_buf(xdrs, &creds->ctx_handle))
82 		return (FALSE);
83 	return (TRUE);
84 }
85 
86 bool_t
87 __xdr_rpc_gss_init_arg(xdrs, init_arg)
88 	XDR			*xdrs;
89 	rpc_gss_init_arg	*init_arg;
90 {
91 	if (!__xdr_gss_buf(xdrs, init_arg))
92 		return (FALSE);
93 	return (TRUE);
94 }
95 
96 bool_t
97 __xdr_rpc_gss_init_res(xdrs, init_res)
98 	XDR			*xdrs;
99 	rpc_gss_init_res	*init_res;
100 {
101 	if (!__xdr_gss_buf(xdrs, &init_res->ctx_handle) ||
102 			!xdr_u_int(xdrs, (u_int *)&init_res->gss_major) ||
103 			!xdr_u_int(xdrs, (u_int *)&init_res->gss_minor) ||
104 			!xdr_u_int(xdrs, (u_int *)&init_res->seq_window) ||
105 			!__xdr_gss_buf(xdrs, &init_res->token))
106 		return (FALSE);
107 	return (TRUE);
108 }
109 
110 /*
111  * Generic routine to wrap data used by client and server sides.
112  */
113 bool_t
114 __rpc_gss_wrap_data(service, qop, context, seq_num, out_xdrs, xdr_func,
115 							xdr_ptr)
116 	OM_uint32		qop;
117 	rpc_gss_service_t	service;
118 	gss_ctx_id_t		context;
119 	u_int			seq_num;
120 	XDR			*out_xdrs;
121 	bool_t			(*xdr_func)();
122 	caddr_t			xdr_ptr;
123 {
124 	OM_uint32		minor;
125 	gss_buffer_desc		in_buf, out_buf;
126 	XDR			temp_xdrs;
127 	bool_t			conf_state;
128 	bool_t			ret = FALSE;
129 	u_int			bufsiz;
130 	char			*buf;
131 
132 	/*
133 	 * Create a temporary XDR/buffer to hold the data to be wrapped.
134 	 */
135 	out_buf.length = 0;
136 	bufsiz = xdr_sizeof(xdr_func, xdr_ptr) +
137 		xdr_sizeof(xdr_u_int, &seq_num);
138 	if ((buf = (char *)malloc(bufsiz)) == NULL) {
139 		fprintf(stderr, dgettext(TEXT_DOMAIN, "malloc failed in "
140 			"__rpc_gss_wrap_data\n"));
141 		return (FALSE);
142 	}
143 	xdrmem_create(&temp_xdrs, buf, bufsiz, XDR_ENCODE);
144 
145 	/*
146 	 * serialize the sequence number into tmp memory
147 	 */
148 	if (!xdr_u_int(&temp_xdrs, &seq_num))
149 		goto fail;
150 
151 	/*
152 	 * serialize the arguments into tmp memory
153 	 */
154 	if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
155 		goto fail;
156 
157 	/*
158 	 * Data to be wrapped goes in in_buf.  If privacy is used,
159 	 * out_buf will have wrapped data (in_buf will no longer be
160 	 * needed).  If integrity is used, out_buf will have checksum
161 	 * which will follow the data in in_buf.
162 	 */
163 	in_buf.length = xdr_getpos(&temp_xdrs);
164 	in_buf.value = temp_xdrs.x_base;
165 
166 	switch (service) {
167 	case rpc_gss_svc_privacy:
168 		if (gss_seal(&minor, context, TRUE, qop, &in_buf,
169 				&conf_state, &out_buf) != GSS_S_COMPLETE)
170 			goto fail;
171 		in_buf.length = 0;	/* in_buf not needed */
172 		if (!conf_state)
173 			goto fail;
174 		break;
175 	case rpc_gss_svc_integrity:
176 		if (gss_sign(&minor, context, qop, &in_buf,
177 						&out_buf) != GSS_S_COMPLETE)
178 			goto fail;
179 		break;
180 	default:
181 		goto fail;
182 	}
183 
184 	/*
185 	 * write out in_buf and out_buf as needed
186 	 */
187 	if (in_buf.length != 0) {
188 		if (!__xdr_gss_buf(out_xdrs, &in_buf))
189 			goto fail;
190 	}
191 
192 	if (!__xdr_gss_buf(out_xdrs, &out_buf))
193 		goto fail;
194 	ret = TRUE;
195 fail:
196 	XDR_DESTROY(&temp_xdrs);
197 	if (buf)
198 		(void) free(buf);
199 	if (out_buf.length != 0)
200 		(void) gss_release_buffer(&minor, &out_buf);
201 	return (ret);
202 }
203 
204 /*
205  * Generic routine to unwrap data used by client and server sides.
206  */
207 bool_t
208 __rpc_gss_unwrap_data(service, context, seq_num, qop_check, in_xdrs, xdr_func,
209 								xdr_ptr)
210 	rpc_gss_service_t	service;
211 	gss_ctx_id_t		context;
212 	u_int			seq_num;
213 	OM_uint32		qop_check;
214 	XDR			*in_xdrs;
215 	bool_t			(*xdr_func)();
216 	caddr_t			xdr_ptr;
217 {
218 	gss_buffer_desc		in_buf, out_buf;
219 	XDR			temp_xdrs;
220 	u_int			seq_num2;
221 	bool_t			conf;
222 	OM_uint32		major = GSS_S_COMPLETE, minor = 0;
223 	int			qop;
224 
225 	in_buf.value = NULL;
226 	out_buf.value = NULL;
227 
228 	/*
229 	 * Pull out wrapped data.  For privacy service, this is the
230 	 * encrypted data.  For integrity service, this is the data
231 	 * followed by a checksum.
232 	 */
233 	if (!__xdr_gss_buf(in_xdrs, &in_buf))
234 		return (FALSE);
235 
236 	if (service == rpc_gss_svc_privacy) {
237 		major = gss_unseal(&minor, context, &in_buf, &out_buf, &conf,
238 							&qop);
239 		free(in_buf.value);
240 		if (major != GSS_S_COMPLETE)
241 			return (FALSE);
242 		/*
243 		 * Keep the returned token (unencrypted data) in in_buf.
244 		 */
245 		in_buf.length = out_buf.length;
246 		in_buf.value = out_buf.value;
247 
248 		/*
249 		 * If privacy was not used, or if QOP is not what we are
250 		 * expecting, fail.
251 		 */
252 		if (!conf || qop != qop_check)
253 			goto fail;
254 
255 	} else if (service == rpc_gss_svc_integrity) {
256 		if (!__xdr_gss_buf(in_xdrs, &out_buf))
257 			return (FALSE);
258 		major = gss_verify(&minor, context, &in_buf, &out_buf, &qop);
259 		free(out_buf.value);
260 		if (major != GSS_S_COMPLETE) {
261 			free(in_buf.value);
262 			return (FALSE);
263 		}
264 
265 		/*
266 		 * If QOP is not what we are expecting, fail.
267 		 */
268 		if (qop != qop_check)
269 			goto fail;
270 	}
271 
272 	xdrmem_create(&temp_xdrs, in_buf.value, in_buf.length, XDR_DECODE);
273 
274 	/*
275 	 * The data consists of the sequence number followed by the
276 	 * arguments.  Make sure sequence number is what we are
277 	 * expecting (i.e., the value in the header).
278 	 */
279 	if (!xdr_u_int(&temp_xdrs, &seq_num2))
280 		goto fail;
281 	if (seq_num2 != seq_num)
282 		goto fail;
283 
284 	/*
285 	 * Deserialize the arguments into xdr_ptr, and release in_buf.
286 	 */
287 	if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
288 		goto fail;
289 
290 	if (service == rpc_gss_svc_privacy)
291 		(void) gss_release_buffer(&minor, &in_buf);
292 	else
293 		free(in_buf.value);
294 	XDR_DESTROY(&temp_xdrs);
295 	return (TRUE);
296 fail:
297 	XDR_DESTROY(&temp_xdrs);
298 	if (service == rpc_gss_svc_privacy)
299 		(void) gss_release_buffer(&minor, &in_buf);
300 	else
301 		free(in_buf.value);
302 	return (FALSE);
303 }
304 
305 /*ARGSUSED*/
306 int
307 __find_max_data_length(service, context, qop, max_tp_unit_len)
308 	rpc_gss_service_t service;
309 	gss_ctx_id_t	context;
310 	OM_uint32	qop;
311 	int		max_tp_unit_len;
312 {
313 	int		conf;
314 	OM_uint32	maj_stat = GSS_S_COMPLETE, min_stat = 0;
315 	OM_uint32	max_input_size;
316 	int		ret_val = 0;
317 
318 	if (service == rpc_gss_svc_integrity || service == rpc_gss_svc_default)
319 		conf = 0;
320 	else if (service == rpc_gss_svc_privacy)
321 		conf = 1;
322 	else if (service == rpc_gss_svc_none)
323 		return (max_tp_unit_len);
324 
325 	maj_stat = gss_wrap_size_limit(&min_stat,
326 		context, conf, qop,
327 		max_tp_unit_len, &max_input_size);
328 
329 	/*
330 	 * max_input_size may result in negative value
331 	 */
332 	if (maj_stat == GSS_S_COMPLETE) {
333 		if ((int)max_input_size <= 0)
334 			ret_val = 0;
335 		else
336 			ret_val = (int)(max_input_size);
337 	} else {
338 		fprintf(stderr, dgettext(TEXT_DOMAIN,
339 					"gss_wrap_size_limit failed in "
340 					"__find_max_data_length\n"));
341 	}
342 
343 	return (ret_val);
344 }
345