xref: /titanic_50/usr/src/lib/smbsrv/libmlrpc/common/ndr_marshal.c (revision 3e5bc1d795e8c41f3680a71e3954e72d079ee46d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <strings.h>
27 #include <sys/param.h>
28 
29 #include <smbsrv/libsmb.h>
30 #include <smbsrv/libmlrpc.h>
31 
32 #ifdef _BIG_ENDIAN
33 static const int ndr_native_byte_order = NDR_REPLAB_INTG_BIG_ENDIAN;
34 #else
35 static const int ndr_native_byte_order = NDR_REPLAB_INTG_LITTLE_ENDIAN;
36 #endif
37 
38 int
39 ndr_encode_decode_common(ndr_xa_t *mxa, int mode, unsigned opnum,
40     ndr_typeinfo_t *ti, void *datum)
41 {
42 	ndr_stream_t	*nds;
43 	int		m_op = NDR_MODE_TO_M_OP(mode);
44 	int		rc;
45 
46 	if (m_op == NDR_M_OP_MARSHALL)
47 		nds = &mxa->send_nds;
48 	else
49 		nds = &mxa->recv_nds;
50 
51 	/*
52 	 * Make sure that nds is in the correct mode
53 	 */
54 	if (!NDR_MODE_MATCH(nds, mode))
55 		return (NDR_DRC_FAULT_MODE_MISMATCH);
56 
57 	/*
58 	 * Perform the (un)marshalling
59 	 */
60 	if (ndo_operation(nds, ti, opnum, datum))
61 		return (NDR_DRC_OK);
62 
63 	switch (nds->error) {
64 	case NDR_ERR_MALLOC_FAILED:
65 		rc = NDR_DRC_FAULT_OUT_OF_MEMORY;
66 		break;
67 
68 	case NDR_ERR_SWITCH_VALUE_INVALID:
69 		rc = NDR_DRC_FAULT_PARAM_0_INVALID;
70 		break;
71 
72 	case NDR_ERR_UNDERFLOW:
73 		rc = NDR_DRC_FAULT_RECEIVED_RUNT;
74 		break;
75 
76 	case NDR_ERR_GROW_FAILED:
77 		rc = NDR_DRC_FAULT_ENCODE_TOO_BIG;
78 		break;
79 
80 	default:
81 		if (m_op == NDR_M_OP_MARSHALL)
82 			rc = NDR_DRC_FAULT_ENCODE_FAILED;
83 		else
84 			rc = NDR_DRC_FAULT_DECODE_FAILED;
85 		break;
86 	}
87 
88 	return (rc);
89 }
90 
91 int
92 ndr_decode_call(ndr_xa_t *mxa, void *params)
93 {
94 	int rc;
95 
96 	rc = ndr_encode_decode_common(mxa, NDR_MODE_CALL_RECV,
97 	    mxa->opnum, mxa->binding->service->interface_ti, params);
98 
99 	return (rc + NDR_PTYPE_REQUEST);
100 }
101 
102 int
103 ndr_encode_return(ndr_xa_t *mxa, void *params)
104 {
105 	int rc;
106 
107 	rc = ndr_encode_decode_common(mxa, NDR_MODE_RETURN_SEND,
108 	    mxa->opnum, mxa->binding->service->interface_ti, params);
109 
110 	return (rc + NDR_PTYPE_RESPONSE);
111 }
112 
113 int
114 ndr_encode_call(ndr_xa_t *mxa, void *params)
115 {
116 	int rc;
117 
118 	rc = ndr_encode_decode_common(mxa, NDR_MODE_CALL_SEND,
119 	    mxa->opnum, mxa->binding->service->interface_ti, params);
120 
121 	return (rc + NDR_PTYPE_REQUEST);
122 }
123 
124 int
125 ndr_decode_return(ndr_xa_t *mxa, void *params)
126 {
127 	int rc;
128 
129 	rc = ndr_encode_decode_common(mxa, NDR_MODE_RETURN_RECV,
130 	    mxa->opnum, mxa->binding->service->interface_ti, params);
131 
132 	return (rc + NDR_PTYPE_RESPONSE);
133 }
134 
135 int
136 ndr_decode_pdu_hdr(ndr_xa_t *mxa)
137 {
138 	ndr_common_header_t	*hdr = &mxa->recv_hdr.common_hdr;
139 	ndr_stream_t		*nds = &mxa->recv_nds;
140 	int			ptype;
141 	int			rc;
142 	int			charset;
143 	int			byte_order;
144 
145 	if (nds->m_op != NDR_M_OP_UNMARSHALL)
146 		return (NDR_DRC_PTYPE_RPCHDR(NDR_DRC_FAULT_MODE_MISMATCH));
147 
148 	/*
149 	 * All PDU headers are at least this big
150 	 */
151 	rc = NDS_GROW_PDU(nds, sizeof (ndr_common_header_t), 0);
152 	if (!rc)
153 		return (NDR_DRC_PTYPE_RPCHDR(NDR_DRC_FAULT_RECEIVED_RUNT));
154 
155 	/*
156 	 * Peek at the first eight bytes to figure out what we're doing.
157 	 */
158 	rc = NDS_GET_PDU(nds, 0, 8, (char *)hdr, 0, 0);
159 	if (!rc)
160 		return (NDR_DRC_PTYPE_RPCHDR(NDR_DRC_FAULT_DECODE_FAILED));
161 
162 	/*
163 	 * Verify the protocol version.
164 	 */
165 	if ((hdr->rpc_vers != 5) || (hdr->rpc_vers_minor != 0))
166 		return (NDR_DRC_PTYPE_RPCHDR(NDR_DRC_FAULT_DECODE_FAILED));
167 
168 	/*
169 	 * Check for ASCII as the character set.  This is an ASCII
170 	 * versus EBCDIC option and has nothing to do with Unicode.
171 	 */
172 	charset = hdr->packed_drep.intg_char_rep & NDR_REPLAB_CHAR_MASK;
173 	if (charset != NDR_REPLAB_CHAR_ASCII)
174 		return (NDR_DRC_PTYPE_RPCHDR(NDR_DRC_FAULT_DECODE_FAILED));
175 
176 	/*
177 	 * Set the byte swap flag if the PDU byte-order
178 	 * is different from the local byte-order.
179 	 */
180 	byte_order = hdr->packed_drep.intg_char_rep & NDR_REPLAB_INTG_MASK;
181 	nds->swap = (byte_order != ndr_native_byte_order) ? 1 : 0;
182 
183 	ptype = hdr->ptype;
184 	if (ptype == NDR_PTYPE_REQUEST &&
185 	    (hdr->pfc_flags & NDR_PFC_OBJECT_UUID) != 0) {
186 		ptype = NDR_PTYPE_REQUEST_WITH;	/* fake for sizing */
187 	}
188 
189 	mxa->ptype = hdr->ptype;
190 
191 	rc = ndr_encode_decode_common(mxa,
192 	    NDR_M_OP_AND_DIR_TO_MODE(nds->m_op, nds->dir),
193 	    ptype, &TYPEINFO(ndr_hdr), hdr);
194 
195 	return (NDR_DRC_PTYPE_RPCHDR(rc));
196 }
197 
198 /*
199  * Decode an RPC fragment header.  Use ndr_decode_pdu_hdr() to process
200  * the first fragment header then this function to process additional
201  * fragment headers.
202  */
203 void
204 ndr_decode_frag_hdr(ndr_stream_t *nds, ndr_common_header_t *hdr)
205 {
206 	ndr_common_header_t *tmp;
207 	uint8_t *pdu;
208 	int byte_order;
209 
210 	pdu = (uint8_t *)nds->pdu_base_offset + nds->pdu_scan_offset;
211 	bcopy(pdu, hdr, NDR_RSP_HDR_SIZE);
212 
213 	/*
214 	 * Swap non-byte fields if the PDU byte-order
215 	 * is different from the local byte-order.
216 	 */
217 	byte_order = hdr->packed_drep.intg_char_rep & NDR_REPLAB_INTG_MASK;
218 
219 	if (byte_order != ndr_native_byte_order) {
220 		/*LINTED E_BAD_PTR_CAST_ALIGN*/
221 		tmp = (ndr_common_header_t *)pdu;
222 
223 		nds_bswap(&tmp->frag_length, &hdr->frag_length,
224 		    sizeof (WORD));
225 		nds_bswap(&tmp->auth_length, &hdr->auth_length,
226 		    sizeof (WORD));
227 		nds_bswap(&tmp->call_id, &hdr->call_id, sizeof (DWORD));
228 	}
229 }
230 
231 int
232 ndr_encode_pdu_hdr(ndr_xa_t *mxa)
233 {
234 	ndr_common_header_t	*hdr = &mxa->send_hdr.common_hdr;
235 	ndr_stream_t		*nds = &mxa->send_nds;
236 	int			ptype;
237 	int			rc;
238 
239 	if (nds->m_op != NDR_M_OP_MARSHALL)
240 		return (NDR_DRC_PTYPE_RPCHDR(NDR_DRC_FAULT_MODE_MISMATCH));
241 
242 	ptype = hdr->ptype;
243 	if (ptype == NDR_PTYPE_REQUEST &&
244 	    (hdr->pfc_flags & NDR_PFC_OBJECT_UUID) != 0) {
245 		ptype = NDR_PTYPE_REQUEST_WITH;	/* fake for sizing */
246 	}
247 
248 	rc = ndr_encode_decode_common(mxa,
249 	    NDR_M_OP_AND_DIR_TO_MODE(nds->m_op, nds->dir),
250 	    ptype, &TYPEINFO(ndr_hdr), hdr);
251 
252 	return (NDR_DRC_PTYPE_RPCHDR(rc));
253 }
254 
255 /*
256  * This is a hand-coded derivative of the automatically generated
257  * (un)marshalling routine for bind_ack headers. bind_ack headers
258  * have an interior conformant array, which is inconsistent with
259  * IDL/NDR rules.
260  */
261 extern struct ndr_typeinfo ndt__uchar;
262 extern struct ndr_typeinfo ndt__ushort;
263 extern struct ndr_typeinfo ndt__ulong;
264 
265 int ndr__ndr_bind_ack_hdr(ndr_ref_t *encl_ref);
266 ndr_typeinfo_t ndt__ndr_bind_ack_hdr = {
267     1,		/* NDR version */
268     3,		/* alignment */
269     NDR_F_STRUCT,	/* flags */
270     ndr__ndr_bind_ack_hdr,	/* ndr_func */
271     68,		/* pdu_size_fixed_part */
272     0,		/* pdu_size_variable_part */
273     68,		/* c_size_fixed_part */
274     0,		/* c_size_variable_part */
275 };
276 
277 /*
278  * [_no_reorder]
279  */
280 int
281 ndr__ndr_bind_ack_hdr(ndr_ref_t *encl_ref)
282 {
283 	ndr_stream_t		*nds = encl_ref->stream;
284 	struct ndr_bind_ack_hdr	*val = /*LINTED E_BAD_PTR_CAST_ALIGN*/
285 	    (struct ndr_bind_ack_hdr *)encl_ref->datum;
286 	ndr_ref_t		myref;
287 	unsigned long		offset;
288 
289 	bzero(&myref, sizeof (myref));
290 	myref.enclosing = encl_ref;
291 	myref.stream = encl_ref->stream;
292 	myref.packed_alignment = 0;
293 
294 	/* do all members in order */
295 	NDR_MEMBER(_ndr_common_header, common_hdr, 0UL);
296 	NDR_MEMBER(_ushort, max_xmit_frag, 16UL);
297 	NDR_MEMBER(_ushort, max_recv_frag, 18UL);
298 	NDR_MEMBER(_ulong, assoc_group_id, 20UL);
299 
300 	/* port any is the conformant culprit */
301 	offset = 24UL;
302 
303 	switch (nds->m_op) {
304 	case NDR_M_OP_MARSHALL:
305 		val->sec_addr.length =
306 		    strlen((char *)val->sec_addr.port_spec) + 1;
307 		break;
308 
309 	case NDR_M_OP_UNMARSHALL:
310 		break;
311 
312 	default:
313 		NDR_SET_ERROR(encl_ref, NDR_ERR_M_OP_INVALID);
314 		return (0);
315 	}
316 
317 	NDR_MEMBER(_ushort, sec_addr.length, offset);
318 	NDR_MEMBER_ARR_WITH_DIMENSION(_uchar, sec_addr.port_spec,
319 	    offset+2UL, val->sec_addr.length);
320 
321 	offset += 2;
322 	offset += val->sec_addr.length;
323 	offset += NDR_ALIGN4(offset);
324 
325 	NDR_MEMBER(_ndr_p_result_list, p_result_list, offset);
326 	return (1);
327 }
328 
329 /*
330  * Assume a single presentation context element in the result list.
331  */
332 unsigned
333 ndr_bind_ack_hdr_size(ndr_xa_t *mxa)
334 {
335 	ndr_bind_ack_hdr_t *bahdr = &mxa->send_hdr.bind_ack_hdr;
336 	unsigned	offset;
337 	unsigned	length;
338 
339 	/* port any is the conformant culprit */
340 	offset = 24UL;
341 
342 	length = strlen((char *)bahdr->sec_addr.port_spec) + 1;
343 
344 	offset += 2;
345 	offset += length;
346 	offset += NDR_ALIGN4(offset);
347 	offset += sizeof (ndr_p_result_list_t);
348 	return (offset);
349 }
350 
351 /*
352  * This is a hand-coded derivative of the automatically generated
353  * (un)marshalling routine for alter_context_rsp headers.
354  * Alter context response headers have an interior conformant array,
355  * which is inconsistent with IDL/NDR rules.
356  */
357 int ndr__ndr_alter_context_rsp_hdr(ndr_ref_t *encl_ref);
358 ndr_typeinfo_t ndt__ndr_alter_context_rsp_hdr = {
359     1,			/* NDR version */
360     3,			/* alignment */
361     NDR_F_STRUCT,	/* flags */
362     ndr__ndr_alter_context_rsp_hdr,	/* ndr_func */
363     56,			/* pdu_size_fixed_part */
364     0,			/* pdu_size_variable_part */
365     56,			/* c_size_fixed_part */
366     0,			/* c_size_variable_part */
367 };
368 
369 /*
370  * [_no_reorder]
371  */
372 int
373 ndr__ndr_alter_context_rsp_hdr(ndr_ref_t *encl_ref)
374 {
375 	ndr_stream_t		*nds = encl_ref->stream;
376 	ndr_alter_context_rsp_hdr_t *val = /*LINTED E_BAD_PTR_CAST_ALIGN*/
377 	    (ndr_alter_context_rsp_hdr_t *)encl_ref->datum;
378 	ndr_ref_t		myref;
379 	unsigned long		offset;
380 
381 	bzero(&myref, sizeof (myref));
382 	myref.enclosing = encl_ref;
383 	myref.stream = encl_ref->stream;
384 	myref.packed_alignment = 0;
385 
386 	/* do all members in order */
387 	NDR_MEMBER(_ndr_common_header, common_hdr, 0UL);
388 	NDR_MEMBER(_ushort, max_xmit_frag, 16UL);
389 	NDR_MEMBER(_ushort, max_recv_frag, 18UL);
390 	NDR_MEMBER(_ulong, assoc_group_id, 20UL);
391 
392 	offset = 24UL;	/* offset of sec_addr */
393 
394 	switch (nds->m_op) {
395 	case NDR_M_OP_MARSHALL:
396 		val->sec_addr.length = 0;
397 		break;
398 
399 	case NDR_M_OP_UNMARSHALL:
400 		break;
401 
402 	default:
403 		NDR_SET_ERROR(encl_ref, NDR_ERR_M_OP_INVALID);
404 		return (0);
405 	}
406 
407 	NDR_MEMBER(_ushort, sec_addr.length, offset);
408 	NDR_MEMBER_ARR_WITH_DIMENSION(_uchar, sec_addr.port_spec,
409 	    offset+2UL, val->sec_addr.length);
410 
411 	offset += 2;	/* sizeof (sec_addr.length) */
412 	offset += NDR_ALIGN4(offset);
413 
414 	NDR_MEMBER(_ndr_p_result_list, p_result_list, offset);
415 	return (1);
416 }
417 
418 /*
419  * Assume a single presentation context element in the result list.
420  */
421 unsigned
422 ndr_alter_context_rsp_hdr_size(void)
423 {
424 	unsigned	offset;
425 
426 	offset = 24UL;	/* offset of sec_addr */
427 	offset += 2;	/* sizeof (sec_addr.length) */
428 	offset += NDR_ALIGN4(offset);
429 	offset += sizeof (ndr_p_result_list_t);
430 	return (offset);
431 }
432