xref: /illumos-gate/usr/src/lib/rpcsec_gss/rpcsec_gss_misc.c (revision a0955b86cd77e22e80846428a5065e871b6d8eb8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright (c) 1986-1995, 1997, 2001 by Sun Microsystems, Inc.
24  * All rights reserved.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Copyright 1993 OpenVision Technologies, Inc., All Rights Reserved.
31  *
32  * $Header:
33  * /afs/gza.com/product/secure/rel-eng/src/1.1/rpc/RCS/auth_gssapi_misc.c,v
34  * 1.10 1994/10/27 12:39:23 jik Exp $
35  */
36 
37 #include <stdlib.h>
38 #include <gssapi/gssapi.h>
39 #include <rpc/rpc.h>
40 #include <rpc/rpcsec_defs.h>
41 
42 /*
43  * Miscellaneous XDR routines.
44  */
45 bool_t
46 __xdr_gss_buf(xdrs, buf)
47 	XDR		*xdrs;
48 	gss_buffer_t	buf;
49 {
50 	u_int cast_len, bound_len;
51 
52 	/*
53 	 * We go through this contortion because size_t is a now a ulong,
54 	 * GSS-API uses ulongs.
55 	 */
56 
57 	if (xdrs->x_op != XDR_DECODE) {
58 		bound_len = cast_len = (u_int) buf->length;
59 	} else {
60 		bound_len = (u_int)-1;
61 	}
62 
63 	if (xdr_bytes(xdrs, (char **)&buf->value, &cast_len,
64 	    bound_len) == TRUE) {
65 		if (xdrs->x_op == XDR_DECODE)
66 			buf->length = cast_len;
67 
68 		return (TRUE);
69 	}
70 
71 	return (FALSE);
72 }
73 
74 bool_t
75 __xdr_rpc_gss_creds(xdrs, creds)
76 	XDR			*xdrs;
77 	rpc_gss_creds		*creds;
78 {
79 	if (!xdr_u_int(xdrs, &creds->version) ||
80 				!xdr_u_int(xdrs, &creds->gss_proc) ||
81 				!xdr_u_int(xdrs, &creds->seq_num) ||
82 				!xdr_u_int(xdrs, (u_int *)&creds->service) ||
83 				!__xdr_gss_buf(xdrs, &creds->ctx_handle))
84 		return (FALSE);
85 	return (TRUE);
86 }
87 
88 bool_t
89 __xdr_rpc_gss_init_arg(xdrs, init_arg)
90 	XDR			*xdrs;
91 	rpc_gss_init_arg	*init_arg;
92 {
93 	if (!__xdr_gss_buf(xdrs, init_arg))
94 		return (FALSE);
95 	return (TRUE);
96 }
97 
98 bool_t
99 __xdr_rpc_gss_init_res(xdrs, init_res)
100 	XDR			*xdrs;
101 	rpc_gss_init_res	*init_res;
102 {
103 	if (!__xdr_gss_buf(xdrs, &init_res->ctx_handle) ||
104 			!xdr_u_int(xdrs, (u_int *)&init_res->gss_major) ||
105 			!xdr_u_int(xdrs, (u_int *)&init_res->gss_minor) ||
106 			!xdr_u_int(xdrs, (u_int *)&init_res->seq_window) ||
107 			!__xdr_gss_buf(xdrs, &init_res->token))
108 		return (FALSE);
109 	return (TRUE);
110 }
111 
112 /*
113  * Generic routine to wrap data used by client and server sides.
114  */
115 bool_t
116 __rpc_gss_wrap_data(service, qop, context, seq_num, out_xdrs, xdr_func,
117 							xdr_ptr)
118 	OM_uint32		qop;
119 	rpc_gss_service_t	service;
120 	gss_ctx_id_t		context;
121 	u_int			seq_num;
122 	XDR			*out_xdrs;
123 	bool_t			(*xdr_func)();
124 	caddr_t			xdr_ptr;
125 {
126 	OM_uint32		minor;
127 	gss_buffer_desc		in_buf, out_buf;
128 	XDR			temp_xdrs;
129 	bool_t			conf_state;
130 	bool_t			ret = FALSE;
131 	u_int			bufsiz;
132 	char			*buf;
133 
134 	/*
135 	 * Create a temporary XDR/buffer to hold the data to be wrapped.
136 	 */
137 	out_buf.length = 0;
138 	bufsiz = xdr_sizeof(xdr_func, xdr_ptr) +
139 		xdr_sizeof(xdr_u_int, &seq_num);
140 	if ((buf = (char *)malloc(bufsiz)) == NULL) {
141 		fprintf(stderr, dgettext(TEXT_DOMAIN, "malloc failed in "
142 			"__rpc_gss_wrap_data\n"));
143 		return (FALSE);
144 	}
145 	xdrmem_create(&temp_xdrs, buf, bufsiz, XDR_ENCODE);
146 
147 	/*
148 	 * serialize the sequence number into tmp memory
149 	 */
150 	if (!xdr_u_int(&temp_xdrs, &seq_num))
151 		goto fail;
152 
153 	/*
154 	 * serialize the arguments into tmp memory
155 	 */
156 	if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
157 		goto fail;
158 
159 	/*
160 	 * Data to be wrapped goes in in_buf.  If privacy is used,
161 	 * out_buf will have wrapped data (in_buf will no longer be
162 	 * needed).  If integrity is used, out_buf will have checksum
163 	 * which will follow the data in in_buf.
164 	 */
165 	in_buf.length = xdr_getpos(&temp_xdrs);
166 	in_buf.value = temp_xdrs.x_base;
167 
168 	switch (service) {
169 	case rpc_gss_svc_privacy:
170 		if (gss_seal(&minor, context, TRUE, qop, &in_buf,
171 				&conf_state, &out_buf) != GSS_S_COMPLETE)
172 			goto fail;
173 		in_buf.length = 0;	/* in_buf not needed */
174 		if (!conf_state)
175 			goto fail;
176 		break;
177 	case rpc_gss_svc_integrity:
178 		if (gss_sign(&minor, context, qop, &in_buf,
179 						&out_buf) != GSS_S_COMPLETE)
180 			goto fail;
181 		break;
182 	default:
183 		goto fail;
184 	}
185 
186 	/*
187 	 * write out in_buf and out_buf as needed
188 	 */
189 	if (in_buf.length != 0) {
190 		if (!__xdr_gss_buf(out_xdrs, &in_buf))
191 			goto fail;
192 	}
193 
194 	if (!__xdr_gss_buf(out_xdrs, &out_buf))
195 		goto fail;
196 	ret = TRUE;
197 fail:
198 	XDR_DESTROY(&temp_xdrs);
199 	if (buf)
200 		(void) free(buf);
201 	if (out_buf.length != 0)
202 		(void) gss_release_buffer(&minor, &out_buf);
203 	return (ret);
204 }
205 
206 /*
207  * Generic routine to unwrap data used by client and server sides.
208  */
209 bool_t
210 __rpc_gss_unwrap_data(service, context, seq_num, qop_check, in_xdrs, xdr_func,
211 								xdr_ptr)
212 	rpc_gss_service_t	service;
213 	gss_ctx_id_t		context;
214 	u_int			seq_num;
215 	OM_uint32		qop_check;
216 	XDR			*in_xdrs;
217 	bool_t			(*xdr_func)();
218 	caddr_t			xdr_ptr;
219 {
220 	gss_buffer_desc		in_buf, out_buf;
221 	XDR			temp_xdrs;
222 	u_int			seq_num2;
223 	bool_t			conf;
224 	OM_uint32		major = GSS_S_COMPLETE, minor = 0;
225 	int			qop;
226 
227 	in_buf.value = NULL;
228 	out_buf.value = NULL;
229 
230 	/*
231 	 * Pull out wrapped data.  For privacy service, this is the
232 	 * encrypted data.  For integrity service, this is the data
233 	 * followed by a checksum.
234 	 */
235 	if (!__xdr_gss_buf(in_xdrs, &in_buf))
236 		return (FALSE);
237 
238 	if (service == rpc_gss_svc_privacy) {
239 		major = gss_unseal(&minor, context, &in_buf, &out_buf, &conf,
240 							&qop);
241 		free(in_buf.value);
242 		if (major != GSS_S_COMPLETE)
243 			return (FALSE);
244 		/*
245 		 * Keep the returned token (unencrypted data) in in_buf.
246 		 */
247 		in_buf.length = out_buf.length;
248 		in_buf.value = out_buf.value;
249 
250 		/*
251 		 * If privacy was not used, or if QOP is not what we are
252 		 * expecting, fail.
253 		 */
254 		if (!conf || qop != qop_check)
255 			goto fail;
256 
257 	} else if (service == rpc_gss_svc_integrity) {
258 		if (!__xdr_gss_buf(in_xdrs, &out_buf))
259 			return (FALSE);
260 		major = gss_verify(&minor, context, &in_buf, &out_buf, &qop);
261 		free(out_buf.value);
262 		if (major != GSS_S_COMPLETE) {
263 			free(in_buf.value);
264 			return (FALSE);
265 		}
266 
267 		/*
268 		 * If QOP is not what we are expecting, fail.
269 		 */
270 		if (qop != qop_check)
271 			goto fail;
272 	}
273 
274 	xdrmem_create(&temp_xdrs, in_buf.value, in_buf.length, XDR_DECODE);
275 
276 	/*
277 	 * The data consists of the sequence number followed by the
278 	 * arguments.  Make sure sequence number is what we are
279 	 * expecting (i.e., the value in the header).
280 	 */
281 	if (!xdr_u_int(&temp_xdrs, &seq_num2))
282 		goto fail;
283 	if (seq_num2 != seq_num)
284 		goto fail;
285 
286 	/*
287 	 * Deserialize the arguments into xdr_ptr, and release in_buf.
288 	 */
289 	if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
290 		goto fail;
291 
292 	if (service == rpc_gss_svc_privacy)
293 		(void) gss_release_buffer(&minor, &in_buf);
294 	else
295 		free(in_buf.value);
296 	XDR_DESTROY(&temp_xdrs);
297 	return (TRUE);
298 fail:
299 	XDR_DESTROY(&temp_xdrs);
300 	if (service == rpc_gss_svc_privacy)
301 		(void) gss_release_buffer(&minor, &in_buf);
302 	else
303 		free(in_buf.value);
304 	return (FALSE);
305 }
306 
307 /*ARGSUSED*/
308 int
309 __find_max_data_length(service, context, qop, max_tp_unit_len)
310 	rpc_gss_service_t service;
311 	gss_ctx_id_t	context;
312 	OM_uint32	qop;
313 	int		max_tp_unit_len;
314 {
315 	int		conf;
316 	OM_uint32	maj_stat = GSS_S_COMPLETE, min_stat = 0;
317 	OM_uint32	max_input_size;
318 	int		ret_val = 0;
319 
320 	if (service == rpc_gss_svc_integrity || service == rpc_gss_svc_default)
321 		conf = 0;
322 	else if (service == rpc_gss_svc_privacy)
323 		conf = 1;
324 	else if (service == rpc_gss_svc_none)
325 		return (max_tp_unit_len);
326 
327 	maj_stat = gss_wrap_size_limit(&min_stat,
328 		context, conf, qop,
329 		max_tp_unit_len, &max_input_size);
330 
331 	/*
332 	 * max_input_size may result in negative value
333 	 */
334 	if (maj_stat == GSS_S_COMPLETE) {
335 		if ((int)max_input_size <= 0)
336 			ret_val = 0;
337 		else
338 			ret_val = (int)(max_input_size);
339 	} else {
340 		fprintf(stderr, dgettext(TEXT_DOMAIN,
341 					"gss_wrap_size_limit failed in "
342 					"__find_max_data_length\n"));
343 	}
344 
345 	return (ret_val);
346 }
347