xref: /titanic_50/usr/src/lib/libelfsign/common/elfsignlib.c (revision de3d2ce46fc25c7b67ccbae4afe5f15e5357568f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #define	ELF_TARGET_ALL	/* get definitions of all section flags */
28 
29 #include <sys/types.h>
30 #include <sys/stat.h>
31 #include <fcntl.h>
32 #include <unistd.h>
33 #include <strings.h>
34 #include <stddef.h>
35 #include <stdlib.h>
36 #include <libintl.h>
37 #include <dirent.h>
38 #include <errno.h>
39 #include <libelf.h>
40 #include <gelf.h>
41 #include <sys/mman.h>
42 #include <cryptoutil.h>
43 #include <sha1.h>
44 #include <sys/crypto/elfsign.h>
45 #include <libelfsign.h>
46 
47 #ifndef SHA1_DIGEST_LENGTH
48 #define	SHA1_DIGEST_LENGTH 20
49 #endif /* SHA1_DIGEST_LENGTH */
50 
51 const char SUNW_ELF_SIGNATURE_ID[] =	ELF_SIGNATURE_SECTION;
52 const char OID_sha1WithRSAEncryption[] = "1.2.840.113549.1.1.5";
53 
54 static ELFsign_status_t elfsign_adjustoffsets(ELFsign_t ess,
55     Elf_Scn *scn, uint64_t new_size);
56 static ELFsign_status_t elfsign_verify_esa(ELFsign_t ess,
57     uchar_t *sig, size_t sig_len);
58 static uint32_t elfsign_switch_uint32(uint32_t i);
59 static ELFsign_status_t elfsign_switch(ELFsign_t ess,
60     struct filesignatures *fssp, enum ES_ACTION action);
61 
62 struct filesig_extraction {
63 	filesig_vers_t	fsx_version;
64 	char	*fsx_format;
65 	char	fsx_signer_DN[ELFCERT_MAX_DN_LEN];
66 	size_t	fsx_signer_DN_len;
67 	uchar_t	fsx_signature[SIG_MAX_LENGTH];
68 	size_t	fsx_sig_len;
69 	char	fsx_sig_oid[100];
70 	size_t	fsx_sig_oid_len;
71 	time_t	fsx_time;
72 };
73 
74 static char *
75 version_to_str(filesig_vers_t v)
76 {
77 	char	*ret;
78 
79 	switch (v) {
80 	case FILESIG_VERSION1:
81 		ret = "VERSION1";
82 		break;
83 	case FILESIG_VERSION2:
84 		ret = "VERSION2";
85 		break;
86 	case FILESIG_VERSION3:
87 		ret = "VERSION3";
88 		break;
89 	case FILESIG_VERSION4:
90 		ret = "VERSION4";
91 		break;
92 	default:
93 		ret = "UNKNOWN";
94 		break;
95 	}
96 	return (ret);
97 }
98 
99 /*
100  * Update filesignatures to include the v1/v2 filesig,
101  *	composed of signer DN, signature, and OID.
102  */
103 static struct filesignatures *
104 filesig_insert_dso(struct filesignatures *fssp,
105     filesig_vers_t	version,
106     const char		*dn,
107     int			dn_len,
108     const uchar_t	*sig,
109     int			sig_len,
110     const char		*oid,
111     int			oid_len)
112 {
113 	struct filesig	*fsgp;
114 	char		*fsdatap;
115 
116 	if (oid == NULL) {
117 		/*
118 		 * This OID is used for the rsa_md5_sha1 format signature also.
119 		 * This use is historical, and is hence continued,
120 		 * despite its lack of technical accuracy.
121 		 */
122 		oid = OID_sha1WithRSAEncryption;
123 		oid_len = strlen(oid);
124 	}
125 
126 	/*
127 	 * for now, always insert a single-signature signature block
128 	 */
129 	if (fssp != NULL)
130 		free(fssp);
131 	fssp  = (struct filesignatures *)
132 	    malloc(filesig_ALIGN(sizeof (struct filesignatures) +
133 	    dn_len + sig_len + oid_len));
134 	if (fssp == NULL)
135 		return (fssp);
136 
137 	fssp->filesig_cnt = 1;
138 	fssp->filesig_pad = 0;	/* reserve for future use */
139 
140 	fsgp = &fssp->filesig_sig;
141 	fsgp->filesig_size = sizeof (struct filesig) +
142 	    dn_len + sig_len + oid_len;
143 	fsgp->filesig_version = version;
144 	switch (version) {
145 	case FILESIG_VERSION1:
146 	case FILESIG_VERSION2:
147 		fsgp->filesig_size -= sizeof (struct filesig) -
148 		    offsetof(struct filesig, filesig_v1_data[0]);
149 		fsgp->filesig_v1_dnsize = dn_len;
150 		fsgp->filesig_v1_sigsize = sig_len;
151 		fsgp->filesig_v1_oidsize = oid_len;
152 		fsdatap = &fsgp->filesig_v1_data[0];
153 		break;
154 	case FILESIG_VERSION3:
155 	case FILESIG_VERSION4:
156 		fsgp->filesig_size -= sizeof (struct filesig) -
157 		    offsetof(struct filesig, filesig_v3_data[0]);
158 		fsgp->filesig_v3_time = time(NULL);
159 		fsgp->filesig_v3_dnsize = dn_len;
160 		fsgp->filesig_v3_sigsize = sig_len;
161 		fsgp->filesig_v3_oidsize = oid_len;
162 		fsdatap = &fsgp->filesig_v3_data[0];
163 		break;
164 	default:
165 		cryptodebug("filesig_insert_dso: unknown version: %d",
166 		    version);
167 		free(fssp);
168 		return (NULL);
169 	}
170 	(void) memcpy(fsdatap, dn, dn_len);
171 	fsdatap += dn_len;
172 	(void) memcpy(fsdatap, (char *)sig, sig_len);
173 	fsdatap += sig_len;
174 	(void) memcpy(fsdatap, oid, oid_len);
175 	fsdatap += oid_len;
176 	fsgp = filesig_next(fsgp);
177 	(void) memset(fsdatap, 0, (char *)(fsgp) - fsdatap);
178 
179 	return (fssp);
180 }
181 
182 /*
183  * filesig_extract - extract filesig structure to internal form
184  */
185 static filesig_vers_t
186 filesig_extract(struct filesig *fsgp, struct filesig_extraction *fsxp)
187 {
188 	char	*fsdp;
189 
190 #define	filesig_extract_common(cp, field, data_var, len_var, len_limit)  { \
191 	len_var = len_limit; \
192 	if (len_var > fsgp->field) \
193 		len_var = fsgp->field; \
194 	(void) memcpy(data_var, cp, len_var); \
195 	cp += fsgp->field; }
196 #define	filesig_extract_str(cp, field, data_var, len_var) \
197 	filesig_extract_common(cp, field, data_var, len_var, \
198 	    sizeof (data_var) - 1); \
199 	data_var[len_var] = '\0';
200 #define	filesig_extract_opaque(cp, field, data_var, len_var) \
201 	filesig_extract_common(cp, field, data_var, len_var, sizeof (data_var))
202 
203 	fsxp->fsx_version = fsgp->filesig_version;
204 	cryptodebug("filesig_extract: version=%s",
205 	    version_to_str(fsxp->fsx_version));
206 	switch (fsxp->fsx_version) {
207 	case FILESIG_VERSION1:
208 	case FILESIG_VERSION2:
209 		/*
210 		 * extract VERSION1 DN, signature, and OID
211 		 */
212 		fsdp = fsgp->filesig_v1_data;
213 		fsxp->fsx_format = ES_FMT_RSA_MD5_SHA1;
214 		fsxp->fsx_time = 0;
215 		filesig_extract_str(fsdp, filesig_v1_dnsize,
216 		    fsxp->fsx_signer_DN, fsxp->fsx_signer_DN_len);
217 		filesig_extract_opaque(fsdp, filesig_v1_sigsize,
218 		    fsxp->fsx_signature, fsxp->fsx_sig_len);
219 		filesig_extract_str(fsdp, filesig_v1_oidsize,
220 		    fsxp->fsx_sig_oid, fsxp->fsx_sig_oid_len);
221 		break;
222 	case FILESIG_VERSION3:
223 	case FILESIG_VERSION4:
224 		fsdp = fsgp->filesig_v3_data;
225 		fsxp->fsx_format = ES_FMT_RSA_SHA1;
226 		fsxp->fsx_time = fsgp->filesig_v3_time;
227 		filesig_extract_str(fsdp, filesig_v3_dnsize,
228 		    fsxp->fsx_signer_DN, fsxp->fsx_signer_DN_len);
229 		filesig_extract_opaque(fsdp, filesig_v3_sigsize,
230 		    fsxp->fsx_signature, fsxp->fsx_sig_len);
231 		filesig_extract_str(fsdp, filesig_v3_oidsize,
232 		    fsxp->fsx_sig_oid, fsxp->fsx_sig_oid_len);
233 		break;
234 	default:
235 		break;
236 	}
237 
238 	return (fsxp->fsx_version);
239 }
240 
241 ELFsign_status_t
242 elfsign_begin(const char *filename, enum ES_ACTION action, ELFsign_t *essp)
243 {
244 	Elf_Cmd		elfcmd;
245 	int		oflags = 0;
246 	short		l_type;
247 	ELFsign_t	ess;
248 	struct stat	stb;
249 	union {
250 		char	c[2];
251 		short	s;
252 	}	uorder;
253 	GElf_Ehdr	elfehdr;
254 	char		*ident;
255 
256 	switch (action) {
257 	case ES_GET:
258 	case ES_GET_CRYPTO:
259 		cryptodebug("elfsign_begin for get");
260 		elfcmd = ELF_C_READ;
261 		oflags = O_RDONLY | O_NOCTTY | O_NDELAY;
262 		l_type = F_RDLCK;
263 		break;
264 	case ES_UPDATE_RSA_MD5_SHA1:
265 	case ES_UPDATE_RSA_SHA1:
266 		cryptodebug("elfsign_begin for update");
267 		elfcmd = ELF_C_RDWR;
268 		oflags = O_RDWR | O_NOCTTY | O_NDELAY;
269 		l_type = F_WRLCK;
270 		break;
271 	default:
272 		return (ELFSIGN_UNKNOWN);
273 	}
274 
275 	if ((ess = malloc(sizeof (struct ELFsign_s))) == NULL) {
276 		return (ELFSIGN_UNKNOWN);
277 	}
278 	(void) memset((void *)ess, 0, sizeof (struct ELFsign_s));
279 
280 	if (!elfcertlib_init(ess)) {
281 		cryptodebug("elfsign_begin: failed initialization");
282 		return (ELFSIGN_UNKNOWN);
283 	}
284 
285 	ess->es_elf = NULL;
286 	ess->es_action = action;
287 	ess->es_version = FILESIG_UNKNOWN;
288 	ess->es_pathname = NULL;
289 	ess->es_certpath = NULL;
290 
291 	if (filename == NULL) {
292 		*essp = ess;
293 		return (ELFSIGN_SUCCESS);
294 	}
295 
296 	if ((ess->es_fd = open(filename, oflags)) == -1) {
297 		elfsign_end(ess);
298 		return (ELFSIGN_INVALID_ELFOBJ);
299 	}
300 	if ((fstat(ess->es_fd, &stb) == -1) || !S_ISREG(stb.st_mode)) {
301 		elfsign_end(ess);
302 		return (ELFSIGN_INVALID_ELFOBJ);
303 	}
304 	if ((ess->es_pathname = strdup(filename)) == NULL) {
305 		elfsign_end(ess);
306 		return (ELFSIGN_UNKNOWN);
307 	}
308 	/*
309 	 * The following lock is released in elfsign_end() when we close(2)
310 	 * the es_fd. This ensures that we aren't trying verify a file
311 	 * we are currently updating.
312 	 */
313 	ess->es_flock.l_type = l_type;
314 	ess->es_flock.l_whence = SEEK_CUR;
315 	ess->es_flock.l_start = 0;
316 	ess->es_flock.l_len = 0;
317 	if (fcntl(ess->es_fd, F_SETLK, &ess->es_flock) == -1) {
318 		cryptodebug("fcntl(F_SETLK) of %s failed with: %s",
319 		    ess->es_pathname, strerror(errno));
320 		elfsign_end(ess);
321 		return (ELFSIGN_UNKNOWN);
322 	}
323 
324 	if (elf_version(EV_CURRENT) == EV_NONE) {
325 		elfsign_end(ess);
326 		return (ELFSIGN_UNKNOWN);
327 	}
328 
329 	if ((ess->es_elf = elf_begin(ess->es_fd, elfcmd,
330 	    (Elf *)NULL)) == NULL) {
331 		cryptodebug("elf_begin() failed: %s", elf_errmsg(-1));
332 		elfsign_end(ess);
333 		return (ELFSIGN_INVALID_ELFOBJ);
334 	}
335 
336 	if (gelf_getehdr(ess->es_elf, &elfehdr) == NULL) {
337 		cryptodebug("elf_getehdr() failed: %s", elf_errmsg(-1));
338 		elfsign_end(ess);
339 		return (ELFSIGN_INVALID_ELFOBJ);
340 	}
341 	ess->es_has_phdr = (elfehdr.e_phnum != 0);
342 
343 	uorder.s = ELFDATA2MSB << 8 | ELFDATA2LSB;
344 	ident = elf_getident(ess->es_elf, NULL);
345 	if (ident == NULL) {
346 		cryptodebug("elf_getident() failed: %s", elf_errmsg(-1));
347 		elfsign_end(ess);
348 		return (ELFSIGN_INVALID_ELFOBJ);
349 	}
350 	ess->es_same_endian = (ident[EI_DATA] == uorder.c[0]);
351 	ess->es_ei_class = ident[EI_CLASS];
352 
353 	/*
354 	 * Call elf_getshstrndx to be sure we have a real ELF object
355 	 * this is required because elf_begin doesn't check that.
356 	 */
357 	if (elf_getshstrndx(ess->es_elf, &ess->es_shstrndx) == 0) {
358 		elfsign_end(ess);
359 		cryptodebug("elfsign_begin: elf_getshstrndx failed");
360 		return (ELFSIGN_INVALID_ELFOBJ);
361 	}
362 
363 	/*
364 	 * Make sure libelf doesn't rearrange section ordering / offsets.
365 	 */
366 	(void) elf_flagelf(ess->es_elf, ELF_C_SET, ELF_F_LAYOUT);
367 
368 	*essp = ess;
369 
370 	return (ELFSIGN_SUCCESS);
371 }
372 
373 /*
374  * elfsign_end - cleanup the ELFsign_t
375  *
376  * IN/OUT:	ess
377  */
378 void
379 elfsign_end(ELFsign_t ess)
380 {
381 	if (ess == NULL)
382 		return;
383 
384 	if (ess->es_elf != NULL && ES_ACTISUPDATE(ess->es_action)) {
385 		if (elf_update(ess->es_elf, ELF_C_WRITE) == -1) {
386 			cryptodebug("elf_update() failed: %s",
387 			    elf_errmsg(-1));
388 			return;
389 		}
390 	}
391 
392 	if (ess->es_fd != -1) {
393 		(void) close(ess->es_fd);
394 		ess->es_fd = -1;
395 	}
396 
397 	if (ess->es_pathname != NULL) {
398 		free(ess->es_pathname);
399 		ess->es_pathname = NULL;
400 	}
401 	if (ess->es_certpath != NULL) {
402 		free(ess->es_certpath);
403 		ess->es_certpath = NULL;
404 	}
405 
406 	if (ess->es_elf != NULL) {
407 		(void) elf_end(ess->es_elf);
408 		ess->es_elf = NULL;
409 	}
410 
411 	elfcertlib_fini(ess);
412 
413 	free(ess);
414 }
415 
416 /*
417  * set the certificate path
418  */
419 ELFsign_status_t
420 elfsign_setcertpath(ELFsign_t ess, const char *certpath)
421 {
422 	/*
423 	 * Normally use of access(2) is insecure, here we are only
424 	 * doing it to help provide early failure and better error
425 	 * checking, so there is no race condition.
426 	 */
427 	if (access(certpath, R_OK) != 0)
428 		return (ELFSIGN_INVALID_CERTPATH);
429 
430 	if ((ess->es_certpath = strdup(certpath)) == NULL)
431 		return (ELFSIGN_FAILED);
432 
433 	if (ES_ACTISUPDATE(ess->es_action)) {
434 		ELFCert_t	cert = NULL;
435 		char		*subject;
436 
437 		/* set the version based on the certificate */
438 		if (elfcertlib_getcert(ess, ess->es_certpath, NULL,
439 		    &cert, ess->es_action)) {
440 			if ((subject = elfcertlib_getdn(cert)) != NULL) {
441 				if (strstr(subject, ELFSIGN_CRYPTO))
442 					ess->es_version = (ess->es_action ==
443 					    ES_UPDATE_RSA_MD5_SHA1) ?
444 					    FILESIG_VERSION1 : FILESIG_VERSION3;
445 				else
446 					ess->es_version = (ess->es_action ==
447 					    ES_UPDATE_RSA_MD5_SHA1) ?
448 					    FILESIG_VERSION2 : FILESIG_VERSION4;
449 			}
450 			elfcertlib_releasecert(ess, cert);
451 		}
452 		if (ess->es_version == FILESIG_UNKNOWN)
453 			return (ELFSIGN_FAILED);
454 	}
455 	return (ELFSIGN_SUCCESS);
456 }
457 
458 /*
459  * set the callback context
460  */
461 void
462 elfsign_setcallbackctx(ELFsign_t ess, void *ctx)
463 {
464 	ess->es_callbackctx = ctx;
465 }
466 
467 /*
468  * set the signature extraction callback
469  */
470 void
471 elfsign_setsigvercallback(ELFsign_t ess,
472     void (*cb)(void *, void *, size_t, ELFCert_t))
473 {
474 	ess->es_sigvercallback = cb;
475 }
476 
477 /*
478  * elfsign_signatures
479  *
480  * IN: 	ess, fsspp, action
481  * OUT:	fsspp
482  */
483 ELFsign_status_t
484 elfsign_signatures(ELFsign_t ess,
485     struct filesignatures **fsspp,
486     size_t *fslen,
487     enum ES_ACTION action)
488 {
489 	Elf_Scn		*scn = NULL, *sig_scn = NULL;
490 	GElf_Shdr	shdr;
491 	Elf_Data	*data = NULL;
492 	const char	*elf_section = SUNW_ELF_SIGNATURE_ID;
493 	int		fscnt, fssize;
494 	struct filesig	*fsgp, *fsgpnext;
495 	uint64_t	sig_offset = 0;
496 
497 	cryptodebug("elfsign_signature");
498 	if ((ess == NULL) || (fsspp == NULL)) {
499 		cryptodebug("invalid arguments");
500 		return (ELFSIGN_UNKNOWN);
501 	}
502 
503 	cryptodebug("elfsign_signature %s for %s",
504 	    ES_ACTISUPDATE(action) ? "ES_UPDATE" : "ES_GET", elf_section);
505 
506 	(void) elf_errno();
507 	while ((scn = elf_nextscn(ess->es_elf, scn)) != NULL) {
508 		const char	*sh_name;
509 		/*
510 		 * Do a string compare to examine each section header
511 		 * to see if this is the section that needs to be updated.
512 		 */
513 		if (gelf_getshdr(scn, &shdr) == NULL) {
514 			cryptodebug("gelf_getshdr() failed: %s",
515 			    elf_errmsg(-1));
516 			return (ELFSIGN_FAILED);
517 		}
518 		sh_name = elf_strptr(ess->es_elf, ess->es_shstrndx,
519 		    (size_t)shdr.sh_name);
520 		if (strcmp(sh_name, elf_section) == 0) {
521 			cryptodebug("elfsign_signature: found %s", elf_section);
522 			sig_scn = scn;
523 			break;
524 		}
525 		if (shdr.sh_type != SHT_NOBITS &&
526 		    sig_offset < shdr.sh_offset + shdr.sh_size) {
527 			sig_offset = shdr.sh_offset + shdr.sh_size;
528 		}
529 	}
530 	if (elf_errmsg(0) != NULL) {
531 		cryptodebug("unexpected error: %s", elf_section,
532 		    elf_errmsg(-1));
533 		return (ELFSIGN_FAILED);
534 	}
535 
536 	if (ES_ACTISUPDATE(action) && (sig_scn == NULL))  {
537 		size_t	old_size, new_size;
538 		char	*new_d_buf;
539 
540 		cryptodebug("elfsign_signature: %s not found - creating",
541 		    elf_section);
542 
543 		/*
544 		 * insert section name in .shstrtab
545 		 */
546 		if ((scn = elf_getscn(ess->es_elf, ess->es_shstrndx)) == 0) {
547 			cryptodebug("elf_getscn() failed: %s",
548 			    elf_errmsg(-1));
549 			return (ELFSIGN_FAILED);
550 		}
551 		if (gelf_getshdr(scn, &shdr) == NULL) {
552 			cryptodebug("gelf_getshdr() failed: %s",
553 			    elf_errmsg(-1));
554 			return (ELFSIGN_FAILED);
555 		}
556 		if ((data = elf_getdata(scn, data)) == NULL) {
557 			cryptodebug("elf_getdata() failed: %s",
558 			    elf_errmsg(-1));
559 			return (ELFSIGN_FAILED);
560 		}
561 		old_size = data->d_size;
562 		if (old_size != shdr.sh_size) {
563 			cryptodebug("mismatch between data size %d "
564 			    "and section size %lld", old_size, shdr.sh_size);
565 			return (ELFSIGN_FAILED);
566 		}
567 		new_size = old_size + strlen(elf_section) + 1;
568 		if ((new_d_buf = malloc(new_size)) == NULL)
569 			return (ELFSIGN_FAILED);
570 
571 		(void) memcpy(new_d_buf, data->d_buf, old_size);
572 		(void) strlcpy(new_d_buf + old_size, elf_section,
573 		    new_size - old_size);
574 		data->d_buf = new_d_buf;
575 		data->d_size = new_size;
576 		data->d_align = 1;
577 		/*
578 		 * Add the section name passed in to the end of the file.
579 		 * Initialize the fields in the Section Header that
580 		 * libelf will not fill in.
581 		 */
582 		if ((sig_scn = elf_newscn(ess->es_elf)) == 0) {
583 			cryptodebug("elf_newscn() failed: %s",
584 			    elf_errmsg(-1));
585 			return (ELFSIGN_FAILED);
586 		}
587 		if (gelf_getshdr(sig_scn, &shdr) == 0) {
588 			cryptodebug("gelf_getshdr() failed: %s",
589 			    elf_errmsg(-1));
590 			return (ELFSIGN_FAILED);
591 		}
592 		shdr.sh_name = old_size;
593 		shdr.sh_type = SHT_SUNW_SIGNATURE;
594 		shdr.sh_flags = SHF_EXCLUDE;
595 		shdr.sh_addr = 0;
596 		shdr.sh_link = 0;
597 		shdr.sh_info = 0;
598 		shdr.sh_size = 0;
599 		shdr.sh_offset = sig_offset;
600 		shdr.sh_addralign = 1;
601 
602 		/*
603 		 * Flush the changes to the underlying elf32 or elf64
604 		 * section header.
605 		 */
606 		if (gelf_update_shdr(sig_scn, &shdr) == 0) {
607 			cryptodebug("gelf_update_shdr failed");
608 			return (ELFSIGN_FAILED);
609 		}
610 
611 		if ((data = elf_newdata(sig_scn)) == NULL) {
612 			cryptodebug("can't add elf data area for %s: %s",
613 			    elf_section, elf_errmsg(-1));
614 			return (ELFSIGN_FAILED);
615 		}
616 		if (elfsign_adjustoffsets(ess, scn,
617 		    old_size + strlen(elf_section) + 1) != ELFSIGN_SUCCESS) {
618 			cryptodebug("can't adjust for new section name %s",
619 			    elf_section);
620 			return (ELFSIGN_FAILED);
621 		}
622 	} else {
623 		if (sig_scn == NULL) {
624 			cryptodebug("can't find signature section");
625 			*fsspp = NULL;
626 			return (ELFSIGN_NOTSIGNED);
627 		}
628 		if ((data = elf_getdata(sig_scn, NULL)) == 0) {
629 			cryptodebug("can't get section data for %s",
630 			    elf_section);
631 			return (ELFSIGN_FAILED);
632 		}
633 	}
634 
635 	if (ES_ACTISUPDATE(action))  {
636 		fssize = offsetof(struct filesignatures, _u1);
637 		if (*fsspp != NULL) {
638 			fsgp = &(*fsspp)->filesig_sig;
639 			for (fscnt = 0; fscnt < (*fsspp)->filesig_cnt;
640 			    fscnt++) {
641 				fsgpnext = filesig_next(fsgp);
642 				fssize += (char *)(fsgpnext) - (char *)(fsgp);
643 				fsgp = fsgpnext;
644 			}
645 		}
646 		if (shdr.sh_addr != 0) {
647 			cryptodebug("section %s is part of a loadable segment, "
648 			    "it cannot be changed.\n", elf_section);
649 			return (ELFSIGN_FAILED);
650 		}
651 		if ((data->d_buf = malloc(fssize)) == NULL)
652 			return (ELFSIGN_FAILED);
653 		if (*fsspp != NULL) {
654 			(void) memcpy(data->d_buf, *fsspp, fssize);
655 			(void) elfsign_switch(ess,
656 			    (struct filesignatures *)data->d_buf, action);
657 		}
658 		data->d_size = fssize;
659 		data->d_align = 1;
660 		data->d_type = ELF_T_BYTE;
661 		cryptodebug("elfsign_signature: data->d_size = %d",
662 		    data->d_size);
663 		if (elfsign_adjustoffsets(ess, sig_scn, fssize) !=
664 		    ELFSIGN_SUCCESS) {
665 			cryptodebug("can't adjust for revised signature "
666 			    "section contents");
667 			return (ELFSIGN_FAILED);
668 		}
669 	} else {
670 		*fsspp = malloc(data->d_size);
671 		if (*fsspp == NULL)
672 			return (ELFSIGN_FAILED);
673 		(void) memcpy(*fsspp, data->d_buf, data->d_size);
674 		if (elfsign_switch(ess, *fsspp, ES_GET) != ELFSIGN_SUCCESS) {
675 			free(*fsspp);
676 			*fsspp = NULL;
677 			return (ELFSIGN_FAILED);
678 		}
679 		*fslen = data->d_size;
680 	}
681 
682 	return (ELFSIGN_SUCCESS);
683 }
684 
685 static ELFsign_status_t
686 elfsign_adjustoffsets(ELFsign_t ess, Elf_Scn *scn, uint64_t new_size)
687 {
688 	GElf_Ehdr	elfehdr;
689 	GElf_Shdr	shdr;
690 	uint64_t	prev_end, scn_offset;
691 	char		*name;
692 	Elf_Scn		*scnp;
693 	Elf_Data	*data;
694 	ELFsign_status_t	retval = ELFSIGN_FAILED;
695 	struct scninfo {
696 		struct scninfo	*scni_next;
697 		Elf_Scn		*scni_scn;
698 		uint64_t	scni_offset;
699 	}		*scnip = NULL, *tmpscnip, **scnipp;
700 
701 	/* get the size of the current section */
702 	if (gelf_getshdr(scn, &shdr) == NULL)
703 		return (ELFSIGN_FAILED);
704 	if (shdr.sh_size == new_size)
705 		return (ELFSIGN_SUCCESS);
706 	scn_offset = shdr.sh_offset;
707 	name = elf_strptr(ess->es_elf, ess->es_shstrndx,
708 	    (size_t)shdr.sh_name);
709 	if (shdr.sh_flags & SHF_ALLOC && ess->es_has_phdr) {
710 		cryptodebug("elfsign_adjustoffsets: "
711 		    "can't move allocated section %s", name ? name : "NULL");
712 		return (ELFSIGN_FAILED);
713 	}
714 
715 	/* resize the desired section */
716 	cryptodebug("elfsign_adjustoffsets: "
717 	    "resizing %s at 0x%llx from 0x%llx to 0x%llx",
718 	    name ? name : "NULL", shdr.sh_offset, shdr.sh_size, new_size);
719 	shdr.sh_size = new_size;
720 	if (gelf_update_shdr(scn, &shdr) == 0) {
721 		cryptodebug("gelf_update_shdr failed");
722 		goto bad;
723 	}
724 	prev_end = shdr.sh_offset + shdr.sh_size;
725 
726 	/*
727 	 * find sections whose data follows the changed section
728 	 *	must scan all sections since section data may not
729 	 *	be in same order as section headers
730 	 */
731 	scnp = elf_getscn(ess->es_elf, 0);	/* "seek" to start */
732 	while ((scnp = elf_nextscn(ess->es_elf, scnp)) != NULL) {
733 		if (gelf_getshdr(scnp, &shdr) == NULL)
734 			goto bad;
735 		if (shdr.sh_offset <= scn_offset)
736 			continue;
737 		name = elf_strptr(ess->es_elf, ess->es_shstrndx,
738 		    (size_t)shdr.sh_name);
739 		if (shdr.sh_flags & SHF_ALLOC && ess->es_has_phdr) {
740 			if (shdr.sh_type == SHT_NOBITS) {
741 				/* .bss can occasionally overlap .shrtab */
742 				continue;
743 			}
744 			cryptodebug("elfsign_adjustoffsets: "
745 			    "can't move allocated section %s",
746 			    name ? name : "NULL");
747 			goto bad;
748 		}
749 		/*
750 		 * force reading of data to memory image
751 		 */
752 		data = NULL;
753 		while ((data = elf_rawdata(scnp, data)) != NULL)
754 			;
755 		/*
756 		 * capture section information
757 		 * insert into list in order of sh_offset
758 		 */
759 		cryptodebug("elfsign_adjustoffsets: "
760 		    "may have to adjust section %s, offset 0x%llx",
761 		    name ? name : "NULL", shdr.sh_offset);
762 		tmpscnip = (struct scninfo *)malloc(sizeof (struct scninfo));
763 		if (tmpscnip == NULL) {
764 			cryptodebug("elfsign_adjustoffsets: "
765 			    "memory allocation failure");
766 			goto bad;
767 		}
768 		tmpscnip->scni_scn = scnp;
769 		tmpscnip->scni_offset = shdr.sh_offset;
770 		for (scnipp = &scnip; *scnipp != NULL;
771 		    scnipp = &(*scnipp)->scni_next) {
772 			if ((*scnipp)->scni_offset > tmpscnip->scni_offset)
773 				break;
774 		}
775 		tmpscnip->scni_next = *scnipp;
776 		*scnipp = tmpscnip;
777 	}
778 
779 	/* move following sections as necessary */
780 	for (tmpscnip = scnip; tmpscnip != NULL;
781 	    tmpscnip = tmpscnip->scni_next) {
782 		scnp = tmpscnip->scni_scn;
783 		if (gelf_getshdr(scnp, &shdr) == NULL) {
784 			cryptodebug("elfsign_adjustoffsets: "
785 			    "elf_getshdr for section %d failed",
786 			    elf_ndxscn(scnp));
787 			goto bad;
788 		}
789 		if (shdr.sh_offset >= prev_end)
790 			break;
791 		prev_end = (prev_end + shdr.sh_addralign - 1) &
792 		    (-shdr.sh_addralign);
793 		name = elf_strptr(ess->es_elf, ess->es_shstrndx,
794 		    (size_t)shdr.sh_name);
795 		cryptodebug("elfsign_adjustoffsets: "
796 		    "moving %s size 0x%llx from 0x%llx to 0x%llx",
797 		    name ? name : "NULL", shdr.sh_size,
798 		    shdr.sh_offset, prev_end);
799 		shdr.sh_offset = prev_end;
800 		if (gelf_update_shdr(scnp, &shdr) == 0) {
801 			cryptodebug("gelf_update_shdr failed");
802 			goto bad;
803 		}
804 		prev_end = shdr.sh_offset + shdr.sh_size;
805 	}
806 
807 	/*
808 	 * adjust section header offset in elf header
809 	 */
810 	if (gelf_getehdr(ess->es_elf, &elfehdr) == NULL) {
811 		cryptodebug("elf_getehdr() failed: %s", elf_errmsg(-1));
812 		goto bad;
813 	}
814 	if (elfehdr.e_shoff < prev_end) {
815 		if (ess->es_ei_class == ELFCLASS32)
816 			prev_end = (prev_end + ELF32_FSZ_OFF - 1) &
817 			    (-ELF32_FSZ_OFF);
818 		else if (ess->es_ei_class == ELFCLASS64)
819 			prev_end = (prev_end + ELF64_FSZ_OFF - 1) &
820 			    (-ELF64_FSZ_OFF);
821 		cryptodebug("elfsign_adjustoffsets: "
822 		    "move sh_off from 0x%llx to 0x%llx",
823 		    elfehdr.e_shoff, prev_end);
824 		elfehdr.e_shoff = prev_end;
825 		if (gelf_update_ehdr(ess->es_elf, &elfehdr) == 0) {
826 			cryptodebug("elf_update_ehdr() failed: %s",
827 			    elf_errmsg(-1));
828 			goto bad;
829 		}
830 	}
831 
832 	retval = ELFSIGN_SUCCESS;
833 
834 bad:
835 	while (scnip != NULL) {
836 		tmpscnip = scnip->scni_next;
837 		free(scnip);
838 		scnip = tmpscnip;
839 	}
840 	return (retval);
841 }
842 
843 struct filesignatures *
844 elfsign_insert_dso(ELFsign_t ess,
845     struct filesignatures *fssp,
846     const char		*dn,
847     int			dn_len,
848     const uchar_t	*sig,
849     int			sig_len,
850     const char		*oid,
851     int			oid_len)
852 {
853 	return (filesig_insert_dso(fssp, ess->es_version, dn, dn_len,
854 	    sig, sig_len, oid, oid_len));
855 }
856 
857 /*ARGSUSED*/
858 filesig_vers_t
859 elfsign_extract_sig(ELFsign_t ess,
860     struct filesignatures *fssp,
861     uchar_t		*sig,
862     size_t		*sig_len)
863 {
864 	struct filesig_extraction	fsx;
865 	filesig_vers_t	version;
866 
867 	if (fssp == NULL)
868 		return (FILESIG_UNKNOWN);
869 	if (fssp->filesig_cnt != 1)
870 		return (FILESIG_UNKNOWN);
871 	version = filesig_extract(&fssp->filesig_sig, &fsx);
872 	switch (version) {
873 	case FILESIG_VERSION1:
874 	case FILESIG_VERSION2:
875 	case FILESIG_VERSION3:
876 	case FILESIG_VERSION4:
877 		if (*sig_len >= fsx.fsx_sig_len) {
878 			(void) memcpy((char *)sig, (char *)fsx.fsx_signature,
879 			    *sig_len);
880 			*sig_len = fsx.fsx_sig_len;
881 		} else
882 			version = FILESIG_UNKNOWN;
883 		break;
884 	default:
885 		version = FILESIG_UNKNOWN;
886 		break;
887 	}
888 
889 	if (ess->es_version == FILESIG_UNKNOWN) {
890 		ess->es_version = version;
891 	}
892 
893 	return (version);
894 }
895 
896 static ELFsign_status_t
897 elfsign_hash_common(ELFsign_t ess, uchar_t *hash, size_t *hash_len,
898     boolean_t hash_mem_resident)
899 {
900 	Elf_Scn		*scn = NULL;
901 	ELFsign_status_t elfstat;
902 	GElf_Shdr	shdr;
903 	SHA1_CTX	ctx;
904 
905 	/* The buffer must be large enough to hold the hash */
906 	if (*hash_len < SHA1_DIGEST_LENGTH)
907 		return (ELFSIGN_FAILED);
908 
909 	bzero(hash, *hash_len);
910 
911 	/* Initialize the digest session */
912 	SHA1Init(&ctx);
913 
914 	scn = elf_getscn(ess->es_elf, 0);	/* "seek" to start */
915 	(void) elf_errno();
916 	while ((scn = elf_nextscn(ess->es_elf, scn)) != 0) {
917 		char *name = NULL;
918 		Elf_Data *data = NULL;
919 
920 		if (gelf_getshdr(scn, &shdr) == NULL) {
921 			elfstat = ELFSIGN_FAILED;
922 			goto done;
923 		}
924 
925 		name = elf_strptr(ess->es_elf, ess->es_shstrndx,
926 		    (size_t)shdr.sh_name);
927 		if (name == NULL)
928 			name = "NULL";
929 
930 		if (!hash_mem_resident &&
931 		    (ess->es_version == FILESIG_VERSION1 ||
932 		    ess->es_version == FILESIG_VERSION3)) {
933 			/*
934 			 * skip the signature section only
935 			 */
936 			if (shdr.sh_type == SHT_SUNW_SIGNATURE) {
937 				cryptodebug("elfsign_hash: skipping %s", name);
938 				continue;
939 			}
940 		} else if (!(shdr.sh_flags & SHF_ALLOC)) {
941 			/*
942 			 * select only memory resident sections
943 			 */
944 			cryptodebug("elfsign_hash: skipping %s", name);
945 			continue;
946 		}
947 
948 		/*
949 		 * throw this section into the hash
950 		 *   use elf_rawdata for endian-independence
951 		 *   use elf_getdata to get update of .shstrtab
952 		 */
953 		while ((data = (shdr.sh_type == SHT_STRTAB ?
954 		    elf_getdata(scn, data) : elf_rawdata(scn, data))) != NULL) {
955 			if (data->d_buf == NULL) {
956 				cryptodebug("elfsign_hash: %s has NULL data",
957 				    name);
958 				continue;
959 			}
960 			cryptodebug("elfsign_hash: updating hash "
961 			    "with %s data size=%d", name, data->d_size);
962 			SHA1Update(&ctx, data->d_buf, data->d_size);
963 		}
964 	}
965 	if (elf_errmsg(0) != NULL) {
966 		cryptodebug("elfsign_hash: %s", elf_errmsg(-1));
967 		elfstat = ELFSIGN_FAILED;
968 		goto done;
969 	}
970 
971 	SHA1Final(hash, &ctx);
972 	*hash_len = SHA1_DIGEST_LENGTH;
973 	{ /* DEBUG START */
974 		const int hashstr_len = (*hash_len) * 2 + 1;
975 		char *hashstr = malloc(hashstr_len);
976 
977 		if (hashstr != NULL) {
978 			tohexstr(hash, *hash_len, hashstr, hashstr_len);
979 			cryptodebug("hash value is: %s", hashstr);
980 			free(hashstr);
981 		}
982 	} /* DEBUG END */
983 	elfstat = ELFSIGN_SUCCESS;
984 done:
985 	return (elfstat);
986 }
987 
988 /*
989  * elfsign_hash - return the hash of the ELF sections affecting execution.
990  *
991  * IN:		ess, hash_len
992  * OUT:		hash, hash_len
993  */
994 ELFsign_status_t
995 elfsign_hash(ELFsign_t ess, uchar_t *hash, size_t *hash_len)
996 {
997 	return (elfsign_hash_common(ess, hash, hash_len, B_FALSE));
998 }
999 
1000 /*
1001  * elfsign_hash_mem_resident - return the hash of the ELF sections
1002  * with only memory resident sections.
1003  *
1004  * IN:		ess, hash_len
1005  * OUT:		hash, hash_len
1006  */
1007 ELFsign_status_t
1008 elfsign_hash_mem_resident(ELFsign_t ess, uchar_t *hash, size_t *hash_len)
1009 {
1010 	return (elfsign_hash_common(ess, hash, hash_len, B_TRUE));
1011 }
1012 
1013 /*
1014  * elfsign_hash_esa = return the hash of the esa_buffer
1015  *
1016  * IN:          ess, esa_buf, esa_buf_len, hash_len
1017  * OUT:         hash, hash_len
1018  */
1019 ELFsign_status_t
1020 elfsign_hash_esa(ELFsign_t ess, uchar_t *esa_buf, size_t esa_buf_len,
1021     uchar_t **hash, size_t *hash_len)
1022 {
1023 	SHA1_CTX ctx;
1024 
1025 	cryptodebug("esa_hash version is: %s",
1026 	    version_to_str(ess->es_version));
1027 	if (ess->es_version <= FILESIG_VERSION2) {
1028 		/*
1029 		 * old rsa_md5_sha1 format
1030 		 * signed with MD5 digest, just pass full esa_buf
1031 		 */
1032 		*hash = esa_buf;
1033 		*hash_len = esa_buf_len;
1034 		return (ELFSIGN_SUCCESS);
1035 	}
1036 
1037 	if (*hash_len < SHA1_DIGEST_LENGTH)
1038 		return (ELFSIGN_FAILED);
1039 
1040 	bzero(*hash, *hash_len);
1041 	SHA1Init(&ctx);
1042 	SHA1Update(&ctx, esa_buf, esa_buf_len);
1043 	SHA1Final(*hash, &ctx);
1044 	*hash_len = SHA1_DIGEST_LENGTH;
1045 
1046 	{ /* DEBUG START */
1047 		const int hashstr_len = (*hash_len) * 2 + 1;
1048 		char *hashstr = malloc(hashstr_len);
1049 
1050 		if (hashstr != NULL) {
1051 			tohexstr(*hash, *hash_len, hashstr, hashstr_len);
1052 			cryptodebug("esa_hash value is: %s", hashstr);
1053 			free(hashstr);
1054 		}
1055 	} /* DEBUG END */
1056 
1057 	return (ELFSIGN_SUCCESS);
1058 }
1059 
1060 /*
1061  * elfsign_verify_signature - Verify the signature of the ELF object.
1062  *
1063  * IN:		ess
1064  * OUT:		esipp
1065  * RETURNS:
1066  *	ELFsign_status_t
1067  */
1068 ELFsign_status_t
1069 elfsign_verify_signature(ELFsign_t ess, struct ELFsign_sig_info **esipp)
1070 {
1071 	ELFsign_status_t	ret = ELFSIGN_FAILED;
1072 	struct	filesignatures *fssp;
1073 	struct	filesig *fsgp;
1074 	size_t	fslen;
1075 	struct filesig_extraction	fsx;
1076 	uchar_t	hash[SIG_MAX_LENGTH];
1077 	size_t	hash_len;
1078 	ELFCert_t	cert = NULL;
1079 	int	sigcnt;
1080 	int	nocert = 0;
1081 	struct ELFsign_sig_info	*esip = NULL;
1082 
1083 	if (esipp != NULL) {
1084 		esip = (struct ELFsign_sig_info *)
1085 		    calloc(1, sizeof (struct ELFsign_sig_info));
1086 		*esipp = esip;
1087 	}
1088 
1089 	/*
1090 	 * Find out which cert we need, based on who signed the ELF object
1091 	 */
1092 	if (elfsign_signatures(ess, &fssp, &fslen, ES_GET) != ELFSIGN_SUCCESS) {
1093 		return (ELFSIGN_NOTSIGNED);
1094 	}
1095 
1096 	if (fssp->filesig_cnt < 1) {
1097 		ret = ELFSIGN_FAILED;
1098 		goto cleanup;
1099 	}
1100 
1101 	fsgp = &fssp->filesig_sig;
1102 
1103 	/*
1104 	 * Scan the signature block, looking for a verifiable signature
1105 	 */
1106 	for (sigcnt = 0; sigcnt < fssp->filesig_cnt;
1107 	    sigcnt++, fsgp = filesig_next(fsgp)) {
1108 		ess->es_version = filesig_extract(fsgp, &fsx);
1109 		cryptodebug("elfsign_verify_signature: version=%s",
1110 		    version_to_str(ess->es_version));
1111 		switch (ess->es_version) {
1112 		case FILESIG_VERSION1:
1113 		case FILESIG_VERSION2:
1114 		case FILESIG_VERSION3:
1115 		case FILESIG_VERSION4:
1116 			break;
1117 		default:
1118 			ret = ELFSIGN_FAILED;
1119 			goto cleanup;
1120 		}
1121 
1122 		cryptodebug("elfsign_verify_signature: signer_DN=\"%s\"",
1123 		    fsx.fsx_signer_DN);
1124 		cryptodebug("elfsign_verify_signature: algorithmOID=\"%s\"",
1125 		    fsx.fsx_sig_oid);
1126 		/* return signer DN if requested */
1127 		if (esipp != NULL) {
1128 			esip->esi_format = fsx.fsx_format;
1129 			if (esip->esi_signer != NULL)
1130 				free(esip->esi_signer);
1131 			esip->esi_signer = strdup(fsx.fsx_signer_DN);
1132 			esip->esi_time = fsx.fsx_time;
1133 		}
1134 
1135 		/*
1136 		 * look for certificate
1137 		 */
1138 		if (cert != NULL)
1139 			elfcertlib_releasecert(ess, cert);
1140 
1141 		/*
1142 		 * skip unfound certificates
1143 		 */
1144 		if (!elfcertlib_getcert(ess, ess->es_certpath,
1145 		    fsx.fsx_signer_DN, &cert, ess->es_action)) {
1146 			cryptodebug("unable to find certificate "
1147 			    "with DN=\"%s\" for %s",
1148 			    fsx.fsx_signer_DN, ess->es_pathname);
1149 			nocert++;
1150 			continue;
1151 		}
1152 
1153 		/*
1154 		 * skip unverified certificates
1155 		 *	force verification of crypto certs
1156 		 */
1157 		if ((ess->es_action == ES_GET_CRYPTO ||
1158 		    strstr(fsx.fsx_signer_DN, ELFSIGN_CRYPTO)) &&
1159 		    !elfcertlib_verifycert(ess, cert)) {
1160 			cryptodebug("elfsign_verify_signature: invalid cert");
1161 			nocert++;
1162 			continue;
1163 		}
1164 
1165 		/*
1166 		 * At this time the only sha1WithRSAEncryption is supported,
1167 		 * so check that is what we have and skip with anything else.
1168 		 */
1169 		if (strcmp(fsx.fsx_sig_oid, OID_sha1WithRSAEncryption) != 0) {
1170 			continue;
1171 		}
1172 
1173 		nocert = 0;
1174 		/*
1175 		 * compute file hash
1176 		 */
1177 		hash_len = sizeof (hash);
1178 		if (elfsign_hash(ess, hash, &hash_len) != ELFSIGN_SUCCESS) {
1179 			cryptodebug("elfsign_verify_signature:"
1180 			    " elfsign_hash failed");
1181 			ret = ELFSIGN_FAILED;
1182 			break;
1183 		}
1184 
1185 		{ /* DEBUG START */
1186 			const int sigstr_len = fsx.fsx_sig_len * 2 + 1;
1187 			char *sigstr = malloc(sigstr_len);
1188 
1189 			if (sigstr != NULL) {
1190 				tohexstr(fsx.fsx_signature, fsx.fsx_sig_len,
1191 				    sigstr, sigstr_len);
1192 				cryptodebug("signature value is: %s", sigstr);
1193 				free(sigstr);
1194 			}
1195 		} /* DEBUG END */
1196 
1197 		if (elfcertlib_verifysig(ess, cert,
1198 		    fsx.fsx_signature, fsx.fsx_sig_len, hash, hash_len)) {
1199 			if (ess->es_sigvercallback)
1200 				(ess->es_sigvercallback)
1201 				    (ess->es_callbackctx, fssp, fslen, cert);
1202 			/*
1203 			 * The signature is verified!
1204 			 * Check if this is a restricted provider
1205 			 */
1206 			if (strstr(fsx.fsx_signer_DN, USAGELIMITED) == NULL)
1207 				ret = ELFSIGN_SUCCESS;
1208 			else {
1209 				cryptodebug("DN is tagged for usagelimited");
1210 				ret = elfsign_verify_esa(ess,
1211 				    fsx.fsx_signature, fsx.fsx_sig_len);
1212 			}
1213 			break;
1214 		}
1215 
1216 		cryptodebug("elfsign_verify_signature: invalid signature");
1217 	}
1218 
1219 cleanup:
1220 	if (cert != NULL)
1221 		elfcertlib_releasecert(ess, cert);
1222 
1223 	free(fssp);
1224 	if (ret == ELFSIGN_FAILED && nocert)
1225 		ret = ELFSIGN_INVALID_CERTPATH;
1226 	return (ret);
1227 }
1228 
1229 /*
1230  * Verify the contents of the .esa file, as per Jumbo export control
1231  * document.  Logic in this function should remain unchanged, unless
1232  * a misinterpretation of the jumbo case was found or if there are
1233  * changes in export regulations necessitating a change.
1234  *
1235  * If the .esa file exists, but is somehow corrupted, we just return
1236  * that this is restricted.  This is consistent with the Jumbo export
1237  * case covering this library and other compenents of ON.  Do not change
1238  * this logic without consulting export control.
1239  *
1240  * Please see do_gen_esa() for a description of the esa file format.
1241  *
1242  */
1243 static ELFsign_status_t
1244 elfsign_verify_esa(ELFsign_t ess, uchar_t *orig_sig, size_t orig_sig_len)
1245 {
1246 	ELFsign_status_t ret = ELFSIGN_RESTRICTED;
1247 	char	*elfobj_esa = NULL;
1248 	size_t	elfobj_esa_len;
1249 	int	esa_fd = -1;
1250 	size_t	esa_buf_len = 0;
1251 	uchar_t *main_sig;
1252 	size_t	main_sig_len = 0;
1253 	uchar_t hash[SIG_MAX_LENGTH], *hash_ptr = hash;
1254 	size_t  hash_len = SIG_MAX_LENGTH;
1255 	char 	*esa_dn = NULL;
1256 	size_t	esa_dn_len = 0;
1257 	uchar_t	*esa_sig;
1258 	size_t	esa_sig_len = 0;
1259 	uchar_t *esa_file_buffer = NULL, *esa_file_ptr;
1260 	struct stat statbuf;
1261 	ELFCert_t cert = NULL;
1262 
1263 	cryptodebug("elfsign_verify_esa");
1264 
1265 	/* does the activation file exist? */
1266 	elfobj_esa_len = strlen(ess->es_pathname) + ESA_LEN + 1;
1267 	elfobj_esa = malloc(elfobj_esa_len);
1268 	if (elfobj_esa == NULL) {
1269 		cryptoerror(LOG_STDERR,
1270 		    gettext("Unable to allocate buffer for esa filename."));
1271 		goto cleanup;
1272 	}
1273 
1274 	(void) strlcpy(elfobj_esa, ess->es_pathname, elfobj_esa_len);
1275 	(void) strlcat(elfobj_esa, ESA, elfobj_esa_len);
1276 
1277 	if ((esa_fd = open(elfobj_esa, O_RDONLY|O_NONBLOCK)) == -1) {
1278 		cryptodebug("No .esa file was found, or it was unreadable");
1279 		goto cleanup;
1280 	}
1281 
1282 	cryptodebug("Reading contents of esa file %s", elfobj_esa);
1283 
1284 	if (fstat(esa_fd, &statbuf) == -1) {
1285 		cryptoerror(LOG_STDERR,
1286 		    gettext("Can't stat %s"), elfobj_esa);
1287 		goto cleanup;
1288 	}
1289 
1290 	/*
1291 	 * mmap the buffer to save on syscalls
1292 	 */
1293 	esa_file_buffer = (uchar_t *)mmap(NULL, statbuf.st_size, PROT_READ,
1294 	    MAP_PRIVATE, esa_fd, 0);
1295 
1296 	if (esa_file_buffer == MAP_FAILED) {
1297 		cryptoerror(LOG_STDERR,
1298 		    gettext("Unable to mmap file to a buffer for %s."),
1299 		    elfobj_esa);
1300 		goto cleanup;
1301 	}
1302 
1303 	esa_file_ptr = esa_file_buffer;
1304 	elfsign_buffer_len(ess, &main_sig_len, esa_file_ptr, ES_GET);
1305 	esa_file_ptr += sizeof (uint32_t);
1306 	cryptodebug("Contents of esa file: main_sig_len=%d", main_sig_len);
1307 	main_sig = esa_file_ptr;
1308 
1309 	esa_file_ptr += main_sig_len;
1310 
1311 	/* verify .esa main signature versus original signature */
1312 	if (main_sig_len != orig_sig_len ||
1313 	    memcmp(main_sig, orig_sig, orig_sig_len) != 0) {
1314 		cryptoerror(LOG_STDERR,
1315 		    gettext("Unable to match original signature from %s."),
1316 		    elfobj_esa);
1317 		goto cleanup;
1318 	}
1319 
1320 	elfsign_buffer_len(ess, &esa_dn_len, esa_file_ptr, ES_GET);
1321 	esa_file_ptr += sizeof (uint32_t);
1322 	cryptodebug("Contents of esa file: esa_dn_len=%d", esa_dn_len);
1323 
1324 	esa_dn = malloc(esa_dn_len + 1);
1325 	if (esa_dn == NULL) {
1326 		cryptoerror(LOG_ERR,
1327 		    gettext("Unable to allocate memory for dn buffer."));
1328 		goto cleanup;
1329 	}
1330 	(void) memcpy(esa_dn, esa_file_ptr, esa_dn_len);
1331 	esa_dn[esa_dn_len] = '\0';
1332 	esa_file_ptr += esa_dn_len;
1333 	cryptodebug("Contents of esa file: esa_dn=%s", esa_dn);
1334 
1335 	elfsign_buffer_len(ess, &esa_sig_len, esa_file_ptr, ES_GET);
1336 	esa_file_ptr += sizeof (uint32_t);
1337 	cryptodebug("Contents of esa file: esa_sig_len=%d", esa_sig_len);
1338 
1339 	esa_sig = esa_file_ptr;
1340 
1341 	cryptodebug("Read esa contents, now verifying");
1342 
1343 	/*
1344 	 * dn used in .esa file should not be limited.
1345 	 */
1346 	if (strstr(esa_dn, USAGELIMITED) != NULL) {
1347 		cryptoerror(LOG_ERR,
1348 		    gettext("DN for .esa file is tagged as limited for %s.\n"
1349 		    "Activation files should only be tagged as unlimited.\n"
1350 		    "Please contact vendor for this provider"),
1351 		    ess->es_pathname);
1352 		goto cleanup;
1353 	}
1354 
1355 	if (!elfcertlib_getcert(ess, ess->es_certpath, esa_dn, &cert,
1356 	    ess->es_action)) {
1357 		cryptodebug(gettext("unable to find certificate "
1358 		    "with DN=\"%s\" for %s"),
1359 		    esa_dn, ess->es_pathname);
1360 		goto cleanup;
1361 	}
1362 
1363 	/*
1364 	 * Since we've already matched the original signature
1365 	 * and the main file signature, we can just verify the esa signature
1366 	 * against the main file signature.
1367 	 */
1368 	esa_buf_len = sizeof (uint32_t) + main_sig_len;
1369 
1370 	if (elfsign_hash_esa(ess, esa_file_buffer, esa_buf_len,
1371 	    &hash_ptr, &hash_len) != ELFSIGN_SUCCESS) {
1372 		cryptoerror(LOG_STDERR,
1373 		    gettext("Unable to hash activation contents."));
1374 		goto cleanup;
1375 	}
1376 
1377 
1378 	if (!elfcertlib_verifysig(ess, cert, esa_sig, esa_sig_len,
1379 	    hash_ptr, hash_len)) {
1380 		cryptoerror(LOG_STDERR,
1381 		    gettext("Unable to verify .esa contents for %s"),
1382 		    ess->es_pathname);
1383 		goto cleanup;
1384 	}
1385 
1386 	cryptodebug("Verified esa contents");
1387 	if (ess->es_sigvercallback)
1388 		(ess->es_sigvercallback) (ess->es_callbackctx,
1389 		    esa_file_buffer, statbuf.st_size, cert);
1390 
1391 	/*
1392 	 * validate the certificate used to sign the activation file
1393 	 */
1394 	if (!elfcertlib_verifycert(ess, cert)) {
1395 		cryptoerror(LOG_STDERR,
1396 		    gettext("Unable to verify .esa certificate %s for %s"),
1397 		    esa_dn, ess->es_pathname);
1398 		goto cleanup;
1399 	}
1400 
1401 	cryptodebug("Verified esa certificate");
1402 	ret = ELFSIGN_SUCCESS;
1403 
1404 cleanup:
1405 	if (elfobj_esa != NULL)
1406 		free(elfobj_esa);
1407 
1408 	if (esa_fd != -1)
1409 		(void) close(esa_fd);
1410 
1411 	if (esa_file_buffer != NULL)
1412 		(void) munmap((caddr_t)esa_file_buffer, statbuf.st_size);
1413 
1414 	if (esa_dn != NULL)
1415 		free(esa_dn);
1416 
1417 	if (cert != NULL)
1418 		elfcertlib_releasecert(ess, cert);
1419 
1420 	return (ret);
1421 }
1422 
1423 static uint32_t
1424 elfsign_switch_uint32(uint32_t i)
1425 {
1426 	return (((i & 0xff) << 24) | ((i & 0xff00) << 8) |
1427 	    ((i >> 8) & 0xff00) | ((i >> 24) & 0xff));
1428 }
1429 
1430 static uint64_t
1431 elfsign_switch_uint64(uint64_t i)
1432 {
1433 	return (((uint64_t)elfsign_switch_uint32(i) << 32) |
1434 	    (elfsign_switch_uint32(i >> 32)));
1435 }
1436 
1437 /*
1438  * If appropriate, switch the endianness of the filesignatures structure
1439  *	Examine the structure only when it is in native endianness
1440  */
1441 static ELFsign_status_t
1442 elfsign_switch(ELFsign_t ess, struct filesignatures *fssp,
1443     enum ES_ACTION action)
1444 {
1445 	int		fscnt;
1446 	filesig_vers_t	version;
1447 	struct filesig	*fsgp, *fsgpnext;
1448 
1449 	if (ess->es_same_endian)
1450 		return (ELFSIGN_SUCCESS);
1451 
1452 	if (ES_ACTISUPDATE(action))
1453 		fscnt = fssp->filesig_cnt;
1454 	fssp->filesig_cnt = elfsign_switch_uint32(fssp->filesig_cnt);
1455 	if (!ES_ACTISUPDATE(action))
1456 		fscnt = fssp->filesig_cnt;
1457 
1458 	fsgp = &(fssp)->filesig_sig;
1459 	for (; fscnt > 0; fscnt--, fsgp = fsgpnext) {
1460 		if (ES_ACTISUPDATE(action)) {
1461 			version = fsgp->filesig_version;
1462 			fsgpnext = filesig_next(fsgp);
1463 		}
1464 		fsgp->filesig_size =
1465 		    elfsign_switch_uint32(fsgp->filesig_size);
1466 		fsgp->filesig_version =
1467 		    elfsign_switch_uint32(fsgp->filesig_version);
1468 		if (!ES_ACTISUPDATE(action)) {
1469 			version = fsgp->filesig_version;
1470 			fsgpnext = filesig_next(fsgp);
1471 		}
1472 		switch (version) {
1473 		case FILESIG_VERSION1:
1474 		case FILESIG_VERSION2:
1475 			fsgp->filesig_v1_dnsize =
1476 			    elfsign_switch_uint32(fsgp->filesig_v1_dnsize);
1477 			fsgp->filesig_v1_sigsize =
1478 			    elfsign_switch_uint32(fsgp->filesig_v1_sigsize);
1479 			fsgp->filesig_v1_oidsize =
1480 			    elfsign_switch_uint32(fsgp->filesig_v1_oidsize);
1481 			break;
1482 		case FILESIG_VERSION3:
1483 		case FILESIG_VERSION4:
1484 			fsgp->filesig_v3_time =
1485 			    elfsign_switch_uint64(fsgp->filesig_v3_time);
1486 			fsgp->filesig_v3_dnsize =
1487 			    elfsign_switch_uint32(fsgp->filesig_v3_dnsize);
1488 			fsgp->filesig_v3_sigsize =
1489 			    elfsign_switch_uint32(fsgp->filesig_v3_sigsize);
1490 			fsgp->filesig_v3_oidsize =
1491 			    elfsign_switch_uint32(fsgp->filesig_v3_oidsize);
1492 			break;
1493 		default:
1494 			cryptodebug("elfsign_switch: failed");
1495 			return (ELFSIGN_FAILED);
1496 		}
1497 	}
1498 	return (ELFSIGN_SUCCESS);
1499 }
1500 
1501 /*
1502  * get/put an integer value from/to a buffer, possibly of opposite endianness
1503  */
1504 void
1505 elfsign_buffer_len(ELFsign_t ess, size_t *ip, uchar_t *cp,
1506     enum ES_ACTION action)
1507 {
1508 	uint32_t tmp;
1509 
1510 	if (!ES_ACTISUPDATE(action)) {
1511 		/* fetch integer from buffer */
1512 		(void) memcpy(&tmp, cp, sizeof (tmp));
1513 		if (!ess->es_same_endian) {
1514 			tmp = elfsign_switch_uint32(tmp);
1515 		}
1516 		*ip = tmp;
1517 	} else {
1518 		/* put integer into buffer */
1519 		tmp = *ip;
1520 		if (!ess->es_same_endian) {
1521 			tmp = elfsign_switch_uint32(tmp);
1522 		}
1523 		(void) memcpy(cp, &tmp, sizeof (tmp));
1524 	}
1525 }
1526 
1527 char const *
1528 elfsign_strerror(ELFsign_status_t elferror)
1529 {
1530 	char const *msg = NULL;
1531 
1532 	switch (elferror) {
1533 		case ELFSIGN_SUCCESS:
1534 			msg = gettext("sign or verify of ELF object succeeded");
1535 			break;
1536 		case ELFSIGN_FAILED:
1537 			msg = gettext("sign or verify of ELF object failed");
1538 			break;
1539 		case ELFSIGN_NOTSIGNED:
1540 			msg = gettext("ELF object not signed");
1541 			break;
1542 		case ELFSIGN_INVALID_CERTPATH:
1543 			msg = gettext("cannot access certificate");
1544 			break;
1545 		case ELFSIGN_INVALID_ELFOBJ:
1546 			msg = gettext("unable to open as an ELF object");
1547 			break;
1548 		case ELFSIGN_RESTRICTED:
1549 			msg = gettext("ELF object is restricted");
1550 			break;
1551 		case ELFSIGN_UNKNOWN:
1552 		default:
1553 			msg = gettext("Unknown error");
1554 			break;
1555 	}
1556 
1557 	return (msg);
1558 }
1559 
1560 boolean_t
1561 elfsign_sig_info(struct filesignatures *fssp, struct ELFsign_sig_info **esipp)
1562 {
1563 	struct filesig_extraction	fsx;
1564 	struct ELFsign_sig_info	*esip;
1565 
1566 	esip = (struct ELFsign_sig_info *)
1567 	    calloc(1, sizeof (struct ELFsign_sig_info));
1568 	*esipp = esip;
1569 	if (esip == NULL)
1570 		return (B_FALSE);
1571 
1572 	switch (filesig_extract(&fssp->filesig_sig, &fsx)) {
1573 	case FILESIG_VERSION1:
1574 	case FILESIG_VERSION2:
1575 	case FILESIG_VERSION3:
1576 	case FILESIG_VERSION4:
1577 		esip->esi_format = fsx.fsx_format;
1578 		esip->esi_signer = strdup(fsx.fsx_signer_DN);
1579 		esip->esi_time = fsx.fsx_time;
1580 		break;
1581 	default:
1582 		free(esip);
1583 		*esipp = NULL;
1584 	}
1585 
1586 	return (*esipp != NULL);
1587 }
1588 
1589 void
1590 elfsign_sig_info_free(struct ELFsign_sig_info *esip)
1591 {
1592 	if (esip != NULL) {
1593 		free(esip->esi_signer);
1594 		free(esip);
1595 	}
1596 }
1597