xref: /titanic_44/usr/src/lib/smbsrv/libmlrpc/common/ndr_marshal.c (revision 2b4a78020b9c38d1b95e2f3fefa6d6e4be382d1f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <strings.h>
27 #include <sys/param.h>
28 
29 #include <smbsrv/libsmb.h>
30 #include <smbsrv/ndr.h>
31 #include <smbsrv/mlrpc.h>
32 
33 #ifdef _BIG_ENDIAN
34 static const int mlrpc_native_byte_order = MLRPC_REPLAB_INTG_BIG_ENDIAN;
35 #else
36 static const int mlrpc_native_byte_order = MLRPC_REPLAB_INTG_LITTLE_ENDIAN;
37 #endif
38 
39 int
40 mlrpc_encode_decode_common(struct mlrpc_xaction *mxa, int mode, unsigned opnum,
41     struct ndr_typeinfo *ti, void *datum)
42 {
43 	struct mlndr_stream	*mlnds;
44 	int			m_op = NDR_MODE_TO_M_OP(mode);
45 	int			rc;
46 
47 	if (m_op == NDR_M_OP_MARSHALL)
48 		mlnds = &mxa->send_mlnds;
49 	else
50 		mlnds = &mxa->recv_mlnds;
51 
52 	/*
53 	 * Make sure that mlnds is in the correct mode
54 	 */
55 	if (!NDR_MODE_MATCH(mlnds, mode))
56 		return (MLRPC_DRC_FAULT_MODE_MISMATCH);
57 
58 	/*
59 	 * Perform the (un)marshalling
60 	 */
61 	if (mlndo_operation(mlnds, ti, opnum, datum))
62 		return (MLRPC_DRC_OK);
63 
64 	switch (mlnds->error) {
65 	case NDR_ERR_MALLOC_FAILED:
66 		rc = MLRPC_DRC_FAULT_OUT_OF_MEMORY;
67 		break;
68 
69 	case NDR_ERR_SWITCH_VALUE_INVALID:
70 		rc = MLRPC_DRC_FAULT_PARAM_0_INVALID;
71 		break;
72 
73 	case NDR_ERR_UNDERFLOW:
74 		rc = MLRPC_DRC_FAULT_RECEIVED_RUNT;
75 		break;
76 
77 	case NDR_ERR_GROW_FAILED:
78 		rc = MLRPC_DRC_FAULT_ENCODE_TOO_BIG;
79 		break;
80 
81 	default:
82 		if (m_op == NDR_M_OP_MARSHALL)
83 			rc = MLRPC_DRC_FAULT_ENCODE_FAILED;
84 		else
85 			rc = MLRPC_DRC_FAULT_DECODE_FAILED;
86 		break;
87 	}
88 
89 	return (rc);
90 }
91 
92 int
93 mlrpc_decode_call(struct mlrpc_xaction *mxa, void *params)
94 {
95 	int rc;
96 
97 	rc = mlrpc_encode_decode_common(mxa, NDR_MODE_CALL_RECV,
98 	    mxa->opnum, mxa->binding->service->interface_ti, params);
99 
100 	return (rc + MLRPC_PTYPE_REQUEST);
101 }
102 
103 int
104 mlrpc_encode_return(struct mlrpc_xaction *mxa, void *params)
105 {
106 	int rc;
107 
108 	rc = mlrpc_encode_decode_common(mxa, NDR_MODE_RETURN_SEND,
109 	    mxa->opnum, mxa->binding->service->interface_ti, params);
110 
111 	return (rc + MLRPC_PTYPE_RESPONSE);
112 }
113 
114 int
115 mlrpc_encode_call(struct mlrpc_xaction *mxa, void *params)
116 {
117 	int rc;
118 
119 	rc = mlrpc_encode_decode_common(mxa, NDR_MODE_CALL_SEND,
120 	    mxa->opnum, mxa->binding->service->interface_ti, params);
121 
122 	return (rc + MLRPC_PTYPE_REQUEST);
123 }
124 
125 int
126 mlrpc_decode_return(struct mlrpc_xaction *mxa, void *params)
127 {
128 	int rc;
129 
130 	rc = mlrpc_encode_decode_common(mxa, NDR_MODE_RETURN_RECV,
131 	    mxa->opnum, mxa->binding->service->interface_ti, params);
132 
133 	return (rc + MLRPC_PTYPE_RESPONSE);
134 }
135 
136 int
137 mlrpc_decode_pdu_hdr(struct mlrpc_xaction *mxa)
138 {
139 	ndr_common_header_t *hdr = &mxa->recv_hdr.common_hdr;
140 	struct mlndr_stream 	*mlnds = &mxa->recv_mlnds;
141 	int			ptype;
142 	int			rc;
143 	int			charset;
144 	int			byte_order;
145 
146 	if (mlnds->m_op != NDR_M_OP_UNMARSHALL)
147 		return (MLRPC_DRC_FAULT_MODE_MISMATCH + 0xFF);
148 
149 	/*
150 	 * All PDU headers are at least this big
151 	 */
152 	rc = MLNDS_GROW_PDU(mlnds, sizeof (ndr_common_header_t), 0);
153 	if (!rc)
154 		return (MLRPC_DRC_FAULT_RECEIVED_RUNT + 0xFF);
155 
156 	/*
157 	 * Peek at the first eight bytes to figure out what we're doing.
158 	 */
159 	rc = MLNDS_GET_PDU(mlnds, 0, 8, (char *)hdr, 0, 0);
160 	if (!rc)
161 		return (MLRPC_DRC_FAULT_DECODE_FAILED + 0xFF);
162 
163 	/*
164 	 * Verify the protocol version.
165 	 */
166 	if ((hdr->rpc_vers != 5) || (hdr->rpc_vers_minor != 0))
167 		return (MLRPC_DRC_FAULT_DECODE_FAILED + 0xFF);
168 
169 	/*
170 	 * Check for ASCII as the character set.  This is an ASCII
171 	 * versus EBCDIC option and has nothing to do with Unicode.
172 	 */
173 	charset = hdr->packed_drep.intg_char_rep & MLRPC_REPLAB_CHAR_MASK;
174 	if (charset != MLRPC_REPLAB_CHAR_ASCII)
175 		return (MLRPC_DRC_FAULT_DECODE_FAILED + 0xFF);
176 
177 	/*
178 	 * Set the byte swap flag if the PDU byte-order
179 	 * is different from the local byte-order.
180 	 */
181 	byte_order = hdr->packed_drep.intg_char_rep & MLRPC_REPLAB_INTG_MASK;
182 	mlnds->swap = (byte_order != mlrpc_native_byte_order) ? 1 : 0;
183 
184 	ptype = hdr->ptype;
185 	if (ptype == MLRPC_PTYPE_REQUEST &&
186 	    (hdr->pfc_flags & MLRPC_PFC_OBJECT_UUID) != 0) {
187 		ptype = MLRPC_PTYPE_REQUEST_WITH;	/* fake for sizing */
188 	}
189 
190 	mxa->ptype = hdr->ptype;
191 
192 	rc = mlrpc_encode_decode_common(mxa,
193 	    NDR_M_OP_AND_DIR_TO_MODE(mlnds->m_op, mlnds->dir),
194 	    ptype, &TYPEINFO(ndr_hdr), hdr);
195 
196 	return (rc + 0xFF);
197 }
198 
199 /*
200  * Decode an RPC fragment header.  Use mlrpc_decode_pdu_hdr() to process
201  * the first fragment header then this function to process additional
202  * fragment headers.
203  */
204 void
205 mlrpc_decode_frag_hdr(struct mlndr_stream *mlnds, ndr_common_header_t *hdr)
206 {
207 	ndr_common_header_t *tmp;
208 	uint8_t *pdu;
209 	int byte_order;
210 
211 	pdu = (uint8_t *)mlnds->pdu_base_offset + mlnds->pdu_scan_offset;
212 	bcopy(pdu, hdr, MLRPC_RSP_HDR_SIZE);
213 
214 	/*
215 	 * Swap non-byte fields if the PDU byte-order
216 	 * is different from the local byte-order.
217 	 */
218 	byte_order = hdr->packed_drep.intg_char_rep & MLRPC_REPLAB_INTG_MASK;
219 
220 	if (byte_order != mlrpc_native_byte_order) {
221 		/*LINTED E_BAD_PTR_CAST_ALIGN*/
222 		tmp = (ndr_common_header_t *)pdu;
223 
224 		mlnds_bswap(&tmp->frag_length, &hdr->frag_length,
225 		    sizeof (WORD));
226 		mlnds_bswap(&tmp->auth_length, &hdr->auth_length,
227 		    sizeof (WORD));
228 		mlnds_bswap(&tmp->call_id, &hdr->call_id, sizeof (DWORD));
229 	}
230 }
231 
232 int
233 mlrpc_encode_pdu_hdr(struct mlrpc_xaction *mxa)
234 {
235 	ndr_common_header_t *hdr = &mxa->send_hdr.common_hdr;
236 	struct mlndr_stream 	*mlnds = &mxa->send_mlnds;
237 	int			ptype;
238 	int			rc;
239 
240 	if (mlnds->m_op != NDR_M_OP_MARSHALL)
241 		return (MLRPC_DRC_FAULT_MODE_MISMATCH + 0xFF);
242 
243 	ptype = hdr->ptype;
244 	if (ptype == MLRPC_PTYPE_REQUEST &&
245 	    (hdr->pfc_flags & MLRPC_PFC_OBJECT_UUID) != 0) {
246 		ptype = MLRPC_PTYPE_REQUEST_WITH;	/* fake for sizing */
247 	}
248 
249 	rc = mlrpc_encode_decode_common(mxa,
250 	    NDR_M_OP_AND_DIR_TO_MODE(mlnds->m_op, mlnds->dir),
251 	    ptype, &TYPEINFO(ndr_hdr), hdr);
252 
253 	return (rc + 0xFF);
254 }
255 
256 /*
257  * This is a hand-coded derivative of the automatically generated
258  * (un)marshalling routine for bind_ack headers. bind_ack headers
259  * have an interior conformant array, which is inconsistent with
260  * IDL/NDR rules.
261  */
262 extern struct ndr_typeinfo ndt__uchar;
263 extern struct ndr_typeinfo ndt__ushort;
264 extern struct ndr_typeinfo ndt__ulong;
265 
266 int mlndr__ndr_bind_ack_hdr(struct ndr_reference *encl_ref);
267 struct ndr_typeinfo ndt__ndr_bind_ack_hdr = {
268     1,		/* NDR version */
269     3,		/* alignment */
270     NDR_F_STRUCT,	/* flags */
271     mlndr__ndr_bind_ack_hdr,	/* ndr_func */
272     68,		/* pdu_size_fixed_part */
273     0,		/* pdu_size_variable_part */
274     68,		/* c_size_fixed_part */
275     0,		/* c_size_variable_part */
276 };
277 
278 /*
279  * [_no_reorder]
280  */
281 int
282 mlndr__ndr_bind_ack_hdr(struct ndr_reference *encl_ref)
283 {
284 	struct mlndr_stream 	*mlnds = encl_ref->stream;
285 	struct ndr_bind_ack_hdr	*val = /*LINTED E_BAD_PTR_CAST_ALIGN*/
286 	    (struct ndr_bind_ack_hdr *)encl_ref->datum;
287 	struct ndr_reference	myref;
288 	unsigned long		offset;
289 
290 	bzero(&myref, sizeof (myref));
291 	myref.enclosing = encl_ref;
292 	myref.stream = encl_ref->stream;
293 	myref.packed_alignment = 0;
294 
295 	/* do all members in order */
296 	NDR_MEMBER(_ndr_common_header, common_hdr, 0UL);
297 	NDR_MEMBER(_ushort, max_xmit_frag, 16UL);
298 	NDR_MEMBER(_ushort, max_recv_frag, 18UL);
299 	NDR_MEMBER(_ulong, assoc_group_id, 20UL);
300 
301 	/* port any is the conformant culprit */
302 	offset = 24UL;
303 
304 	switch (mlnds->m_op) {
305 	case NDR_M_OP_MARSHALL:
306 		val->sec_addr.length =
307 		    strlen((char *)val->sec_addr.port_spec) + 1;
308 		break;
309 
310 	case NDR_M_OP_UNMARSHALL:
311 		break;
312 
313 	default:
314 		NDR_SET_ERROR(encl_ref, NDR_ERR_M_OP_INVALID);
315 		return (0);
316 	}
317 
318 	NDR_MEMBER(_ushort, sec_addr.length, offset);
319 	NDR_MEMBER_ARR_WITH_DIMENSION(_uchar, sec_addr.port_spec,
320 	    offset+2UL, val->sec_addr.length);
321 
322 	offset += 2;
323 	offset += val->sec_addr.length;
324 	offset += (4 - offset) & 3;
325 
326 	NDR_MEMBER(_mlrpc_p_result_list, p_result_list, offset);
327 	return (1);
328 }
329 
330 /*
331  * Assume a single presentation context element in the result list.
332  */
333 unsigned
334 mlrpc_bind_ack_hdr_size(struct mlrpc_xaction *mxa)
335 {
336 	ndr_bind_ack_hdr_t *bahdr = &mxa->send_hdr.bind_ack_hdr;
337 	unsigned	offset;
338 	unsigned	length;
339 
340 	/* port any is the conformant culprit */
341 	offset = 24UL;
342 
343 	length = strlen((char *)bahdr->sec_addr.port_spec) + 1;
344 
345 	offset += 2;
346 	offset += length;
347 	offset += (4 - offset) & 3;
348 	offset += sizeof (mlrpc_p_result_list_t);
349 	return (offset);
350 }
351 
352 /*
353  * This is a hand-coded derivative of the automatically generated
354  * (un)marshalling routine for alter_context_rsp headers.
355  * Alter context response headers have an interior conformant array,
356  * which is inconsistent with IDL/NDR rules.
357  */
358 int mlndr__ndr_alter_context_rsp_hdr(struct ndr_reference *encl_ref);
359 struct ndr_typeinfo ndt__ndr_alter_context_rsp_hdr = {
360     1,			/* NDR version */
361     3,			/* alignment */
362     NDR_F_STRUCT,	/* flags */
363     mlndr__ndr_alter_context_rsp_hdr,	/* ndr_func */
364     56,			/* pdu_size_fixed_part */
365     0,			/* pdu_size_variable_part */
366     56,			/* c_size_fixed_part */
367     0,			/* c_size_variable_part */
368 };
369 
370 /*
371  * [_no_reorder]
372  */
373 int
374 mlndr__ndr_alter_context_rsp_hdr(struct ndr_reference *encl_ref)
375 {
376 	struct mlndr_stream 	*mlnds = encl_ref->stream;
377 	ndr_alter_context_rsp_hdr_t *val = /*LINTED E_BAD_PTR_CAST_ALIGN*/
378 	    (ndr_alter_context_rsp_hdr_t *)encl_ref->datum;
379 	struct ndr_reference	myref;
380 	unsigned long		offset;
381 
382 	bzero(&myref, sizeof (myref));
383 	myref.enclosing = encl_ref;
384 	myref.stream = encl_ref->stream;
385 	myref.packed_alignment = 0;
386 
387 	/* do all members in order */
388 	NDR_MEMBER(_ndr_common_header, common_hdr, 0UL);
389 	NDR_MEMBER(_ushort, max_xmit_frag, 16UL);
390 	NDR_MEMBER(_ushort, max_recv_frag, 18UL);
391 	NDR_MEMBER(_ulong, assoc_group_id, 20UL);
392 
393 	offset = 24UL;	/* offset of sec_addr */
394 
395 	switch (mlnds->m_op) {
396 	case NDR_M_OP_MARSHALL:
397 		val->sec_addr.length = 0;
398 		break;
399 
400 	case NDR_M_OP_UNMARSHALL:
401 		break;
402 
403 	default:
404 		NDR_SET_ERROR(encl_ref, NDR_ERR_M_OP_INVALID);
405 		return (0);
406 	}
407 
408 	NDR_MEMBER(_ushort, sec_addr.length, offset);
409 	NDR_MEMBER_ARR_WITH_DIMENSION(_uchar, sec_addr.port_spec,
410 	    offset+2UL, val->sec_addr.length);
411 
412 	offset += 2;	/* sizeof (sec_addr.length) */
413 	offset += (4 - offset) & 3;
414 
415 	NDR_MEMBER(_mlrpc_p_result_list, p_result_list, offset);
416 	return (1);
417 }
418 
419 /*
420  * Assume a single presentation context element in the result list.
421  */
422 unsigned
423 mlrpc_alter_context_rsp_hdr_size(void)
424 {
425 	unsigned	offset;
426 
427 	offset = 24UL;	/* offset of sec_addr */
428 	offset += 2;	/* sizeof (sec_addr.length) */
429 	offset += (4 - offset) & 3;
430 	offset += sizeof (mlrpc_p_result_list_t);
431 	return (offset);
432 }
433