xref: /freebsd/contrib/unbound/validator/val_secalgo.c (revision c66ec88fed842fbaad62c30d510644ceb7bd2d71)
1 /*
2  * validator/val_secalgo.c - validator security algorithm functions.
3  *
4  * Copyright (c) 2012, NLnet Labs. All rights reserved.
5  *
6  * This software is open source.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright notice,
13  * this list of conditions and the following disclaimer.
14  *
15  * Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  *
19  * Neither the name of the NLNET LABS nor the names of its contributors may
20  * be used to endorse or promote products derived from this software without
21  * specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 /**
37  * \file
38  *
39  * This file contains helper functions for the validator module.
40  * These functions take raw data buffers, formatted for crypto verification,
41  * and do the library calls (for the crypto library in use).
42  */
43 #include "config.h"
44 /* packed_rrset on top to define enum types (forced by c99 standard) */
45 #include "util/data/packed_rrset.h"
46 #include "validator/val_secalgo.h"
47 #include "validator/val_nsec3.h"
48 #include "util/log.h"
49 #include "sldns/rrdef.h"
50 #include "sldns/keyraw.h"
51 #include "sldns/sbuffer.h"
52 
53 #if !defined(HAVE_SSL) && !defined(HAVE_NSS) && !defined(HAVE_NETTLE)
54 #error "Need crypto library to do digital signature cryptography"
55 #endif
56 
57 /** fake DSA support for unit tests */
58 int fake_dsa = 0;
59 /** fake SHA1 support for unit tests */
60 int fake_sha1 = 0;
61 
62 /* OpenSSL implementation */
63 #ifdef HAVE_SSL
64 #ifdef HAVE_OPENSSL_ERR_H
65 #include <openssl/err.h>
66 #endif
67 
68 #ifdef HAVE_OPENSSL_RAND_H
69 #include <openssl/rand.h>
70 #endif
71 
72 #ifdef HAVE_OPENSSL_CONF_H
73 #include <openssl/conf.h>
74 #endif
75 
76 #ifdef HAVE_OPENSSL_ENGINE_H
77 #include <openssl/engine.h>
78 #endif
79 
80 #if defined(HAVE_OPENSSL_DSA_H) && defined(USE_DSA)
81 #include <openssl/dsa.h>
82 #endif
83 
84 /**
85  * Output a libcrypto openssl error to the logfile.
86  * @param str: string to add to it.
87  * @param e: the error to output, error number from ERR_get_error().
88  */
89 static void
90 log_crypto_error(const char* str, unsigned long e)
91 {
92 	char buf[128];
93 	/* or use ERR_error_string if ERR_error_string_n is not avail TODO */
94 	ERR_error_string_n(e, buf, sizeof(buf));
95 	/* buf now contains */
96 	/* error:[error code]:[library name]:[function name]:[reason string] */
97 	log_err("%s crypto %s", str, buf);
98 }
99 
100 /* return size of digest if supported, or 0 otherwise */
101 size_t
102 nsec3_hash_algo_size_supported(int id)
103 {
104 	switch(id) {
105 	case NSEC3_HASH_SHA1:
106 		return SHA_DIGEST_LENGTH;
107 	default:
108 		return 0;
109 	}
110 }
111 
112 /* perform nsec3 hash. return false on failure */
113 int
114 secalgo_nsec3_hash(int algo, unsigned char* buf, size_t len,
115         unsigned char* res)
116 {
117 	switch(algo) {
118 	case NSEC3_HASH_SHA1:
119 #ifdef OPENSSL_FIPS
120 		if(!sldns_digest_evp(buf, len, res, EVP_sha1()))
121 			log_crypto_error("could not digest with EVP_sha1",
122 				ERR_get_error());
123 #else
124 		(void)SHA1(buf, len, res);
125 #endif
126 		return 1;
127 	default:
128 		return 0;
129 	}
130 }
131 
132 void
133 secalgo_hash_sha256(unsigned char* buf, size_t len, unsigned char* res)
134 {
135 #ifdef OPENSSL_FIPS
136 	if(!sldns_digest_evp(buf, len, res, EVP_sha256()))
137 		log_crypto_error("could not digest with EVP_sha256",
138 			ERR_get_error());
139 #else
140 	(void)SHA256(buf, len, res);
141 #endif
142 }
143 
144 /**
145  * Return size of DS digest according to its hash algorithm.
146  * @param algo: DS digest algo.
147  * @return size in bytes of digest, or 0 if not supported.
148  */
149 size_t
150 ds_digest_size_supported(int algo)
151 {
152 	switch(algo) {
153 		case LDNS_SHA1:
154 #if defined(HAVE_EVP_SHA1) && defined(USE_SHA1)
155 			return SHA_DIGEST_LENGTH;
156 #else
157 			if(fake_sha1) return 20;
158 			return 0;
159 #endif
160 #ifdef HAVE_EVP_SHA256
161 		case LDNS_SHA256:
162 			return SHA256_DIGEST_LENGTH;
163 #endif
164 #ifdef USE_GOST
165 		case LDNS_HASH_GOST:
166 			/* we support GOST if it can be loaded */
167 			(void)sldns_key_EVP_load_gost_id();
168 			if(EVP_get_digestbyname("md_gost94"))
169 				return 32;
170 			else	return 0;
171 #endif
172 #ifdef USE_ECDSA
173 		case LDNS_SHA384:
174 			return SHA384_DIGEST_LENGTH;
175 #endif
176 		default: break;
177 	}
178 	return 0;
179 }
180 
181 #ifdef USE_GOST
182 /** Perform GOST hash */
183 static int
184 do_gost94(unsigned char* data, size_t len, unsigned char* dest)
185 {
186 	const EVP_MD* md = EVP_get_digestbyname("md_gost94");
187 	if(!md)
188 		return 0;
189 	return sldns_digest_evp(data, (unsigned int)len, dest, md);
190 }
191 #endif
192 
193 int
194 secalgo_ds_digest(int algo, unsigned char* buf, size_t len,
195 	unsigned char* res)
196 {
197 	switch(algo) {
198 #if defined(HAVE_EVP_SHA1) && defined(USE_SHA1)
199 		case LDNS_SHA1:
200 #ifdef OPENSSL_FIPS
201 			if(!sldns_digest_evp(buf, len, res, EVP_sha1()))
202 				log_crypto_error("could not digest with EVP_sha1",
203 					ERR_get_error());
204 #else
205 			(void)SHA1(buf, len, res);
206 #endif
207 			return 1;
208 #endif
209 #ifdef HAVE_EVP_SHA256
210 		case LDNS_SHA256:
211 #ifdef OPENSSL_FIPS
212 			if(!sldns_digest_evp(buf, len, res, EVP_sha256()))
213 				log_crypto_error("could not digest with EVP_sha256",
214 					ERR_get_error());
215 #else
216 			(void)SHA256(buf, len, res);
217 #endif
218 			return 1;
219 #endif
220 #ifdef USE_GOST
221 		case LDNS_HASH_GOST:
222 			if(do_gost94(buf, len, res))
223 				return 1;
224 			break;
225 #endif
226 #ifdef USE_ECDSA
227 		case LDNS_SHA384:
228 #ifdef OPENSSL_FIPS
229 			if(!sldns_digest_evp(buf, len, res, EVP_sha384()))
230 				log_crypto_error("could not digest with EVP_sha384",
231 					ERR_get_error());
232 #else
233 			(void)SHA384(buf, len, res);
234 #endif
235 			return 1;
236 #endif
237 		default:
238 			verbose(VERB_QUERY, "unknown DS digest algorithm %d",
239 				algo);
240 			break;
241 	}
242 	return 0;
243 }
244 
245 /** return true if DNSKEY algorithm id is supported */
246 int
247 dnskey_algo_id_is_supported(int id)
248 {
249 	switch(id) {
250 	case LDNS_RSAMD5:
251 		/* RFC 6725 deprecates RSAMD5 */
252 		return 0;
253 	case LDNS_DSA:
254 	case LDNS_DSA_NSEC3:
255 #if defined(USE_DSA) && defined(USE_SHA1)
256 		return 1;
257 #else
258 		if(fake_dsa || fake_sha1) return 1;
259 		return 0;
260 #endif
261 
262 	case LDNS_RSASHA1:
263 	case LDNS_RSASHA1_NSEC3:
264 #ifdef USE_SHA1
265 		return 1;
266 #else
267 		if(fake_sha1) return 1;
268 		return 0;
269 #endif
270 
271 #if defined(HAVE_EVP_SHA256) && defined(USE_SHA2)
272 	case LDNS_RSASHA256:
273 #endif
274 #if defined(HAVE_EVP_SHA512) && defined(USE_SHA2)
275 	case LDNS_RSASHA512:
276 #endif
277 #ifdef USE_ECDSA
278 	case LDNS_ECDSAP256SHA256:
279 	case LDNS_ECDSAP384SHA384:
280 #endif
281 #ifdef USE_ED25519
282 	case LDNS_ED25519:
283 #endif
284 #ifdef USE_ED448
285 	case LDNS_ED448:
286 #endif
287 #if (defined(HAVE_EVP_SHA256) && defined(USE_SHA2)) || (defined(HAVE_EVP_SHA512) && defined(USE_SHA2)) || defined(USE_ECDSA) || defined(USE_ED25519) || defined(USE_ED448)
288 		return 1;
289 #endif
290 
291 #ifdef USE_GOST
292 	case LDNS_ECC_GOST:
293 		/* we support GOST if it can be loaded */
294 		return sldns_key_EVP_load_gost_id();
295 #endif
296 	default:
297 		return 0;
298 	}
299 }
300 
301 #ifdef USE_DSA
302 /**
303  * Setup DSA key digest in DER encoding ...
304  * @param sig: input is signature output alloced ptr (unless failure).
305  * 	caller must free alloced ptr if this routine returns true.
306  * @param len: input is initial siglen, output is output len.
307  * @return false on failure.
308  */
309 static int
310 setup_dsa_sig(unsigned char** sig, unsigned int* len)
311 {
312 	unsigned char* orig = *sig;
313 	unsigned int origlen = *len;
314 	int newlen;
315 	BIGNUM *R, *S;
316 	DSA_SIG *dsasig;
317 
318 	/* extract the R and S field from the sig buffer */
319 	if(origlen < 1 + 2*SHA_DIGEST_LENGTH)
320 		return 0;
321 	R = BN_new();
322 	if(!R) return 0;
323 	(void) BN_bin2bn(orig + 1, SHA_DIGEST_LENGTH, R);
324 	S = BN_new();
325 	if(!S) return 0;
326 	(void) BN_bin2bn(orig + 21, SHA_DIGEST_LENGTH, S);
327 	dsasig = DSA_SIG_new();
328 	if(!dsasig) return 0;
329 
330 #ifdef HAVE_DSA_SIG_SET0
331 	if(!DSA_SIG_set0(dsasig, R, S)) return 0;
332 #else
333 #  ifndef S_SPLINT_S
334 	dsasig->r = R;
335 	dsasig->s = S;
336 #  endif /* S_SPLINT_S */
337 #endif
338 	*sig = NULL;
339 	newlen = i2d_DSA_SIG(dsasig, sig);
340 	if(newlen < 0) {
341 		DSA_SIG_free(dsasig);
342 		free(*sig);
343 		return 0;
344 	}
345 	*len = (unsigned int)newlen;
346 	DSA_SIG_free(dsasig);
347 	return 1;
348 }
349 #endif /* USE_DSA */
350 
351 #ifdef USE_ECDSA
352 /**
353  * Setup the ECDSA signature in its encoding that the library wants.
354  * Converts from plain numbers to ASN formatted.
355  * @param sig: input is signature, output alloced ptr (unless failure).
356  * 	caller must free alloced ptr if this routine returns true.
357  * @param len: input is initial siglen, output is output len.
358  * @return false on failure.
359  */
360 static int
361 setup_ecdsa_sig(unsigned char** sig, unsigned int* len)
362 {
363         /* convert from two BIGNUMs in the rdata buffer, to ASN notation.
364 	 * ASN preamble: 30440220 <R 32bytefor256> 0220 <S 32bytefor256>
365 	 * the '20' is the length of that field (=bnsize).
366 i	 * the '44' is the total remaining length.
367 	 * if negative, start with leading zero.
368 	 * if starts with 00s, remove them from the number.
369 	 */
370         uint8_t pre[] = {0x30, 0x44, 0x02, 0x20};
371         int pre_len = 4;
372         uint8_t mid[] = {0x02, 0x20};
373         int mid_len = 2;
374         int raw_sig_len, r_high, s_high, r_rem=0, s_rem=0;
375 	int bnsize = (int)((*len)/2);
376         unsigned char* d = *sig;
377 	uint8_t* p;
378 	/* if too short or not even length, fails */
379 	if(*len < 16 || bnsize*2 != (int)*len)
380 		return 0;
381 
382         /* strip leading zeroes from r (but not last one) */
383         while(r_rem < bnsize-1 && d[r_rem] == 0)
384                 r_rem++;
385         /* strip leading zeroes from s (but not last one) */
386         while(s_rem < bnsize-1 && d[bnsize+s_rem] == 0)
387                 s_rem++;
388 
389         r_high = ((d[0+r_rem]&0x80)?1:0);
390         s_high = ((d[bnsize+s_rem]&0x80)?1:0);
391         raw_sig_len = pre_len + r_high + bnsize - r_rem + mid_len +
392                 s_high + bnsize - s_rem;
393 	*sig = (unsigned char*)malloc((size_t)raw_sig_len);
394 	if(!*sig)
395 		return 0;
396 	p = (uint8_t*)*sig;
397 	p[0] = pre[0];
398 	p[1] = (uint8_t)(raw_sig_len-2);
399 	p[2] = pre[2];
400 	p[3] = (uint8_t)(bnsize + r_high - r_rem);
401 	p += 4;
402 	if(r_high) {
403 		*p = 0;
404 		p += 1;
405 	}
406 	memmove(p, d+r_rem, (size_t)bnsize-r_rem);
407 	p += bnsize-r_rem;
408 	memmove(p, mid, (size_t)mid_len-1);
409 	p += mid_len-1;
410 	*p = (uint8_t)(bnsize + s_high - s_rem);
411 	p += 1;
412         if(s_high) {
413 		*p = 0;
414 		p += 1;
415 	}
416 	memmove(p, d+bnsize+s_rem, (size_t)bnsize-s_rem);
417 	*len = (unsigned int)raw_sig_len;
418 	return 1;
419 }
420 #endif /* USE_ECDSA */
421 
422 #ifdef USE_ECDSA_EVP_WORKAROUND
423 static EVP_MD ecdsa_evp_256_md;
424 static EVP_MD ecdsa_evp_384_md;
425 void ecdsa_evp_workaround_init(void)
426 {
427 	/* openssl before 1.0.0 fixes RSA with the SHA256
428 	 * hash in EVP.  We create one for ecdsa_sha256 */
429 	ecdsa_evp_256_md = *EVP_sha256();
430 	ecdsa_evp_256_md.required_pkey_type[0] = EVP_PKEY_EC;
431 	ecdsa_evp_256_md.verify = (void*)ECDSA_verify;
432 
433 	ecdsa_evp_384_md = *EVP_sha384();
434 	ecdsa_evp_384_md.required_pkey_type[0] = EVP_PKEY_EC;
435 	ecdsa_evp_384_md.verify = (void*)ECDSA_verify;
436 }
437 #endif /* USE_ECDSA_EVP_WORKAROUND */
438 
439 /**
440  * Setup key and digest for verification. Adjust sig if necessary.
441  *
442  * @param algo: key algorithm
443  * @param evp_key: EVP PKEY public key to create.
444  * @param digest_type: digest type to use
445  * @param key: key to setup for.
446  * @param keylen: length of key.
447  * @return false on failure.
448  */
449 static int
450 setup_key_digest(int algo, EVP_PKEY** evp_key, const EVP_MD** digest_type,
451 	unsigned char* key, size_t keylen)
452 {
453 #if defined(USE_DSA) && defined(USE_SHA1)
454 	DSA* dsa;
455 #endif
456 	RSA* rsa;
457 
458 	switch(algo) {
459 #if defined(USE_DSA) && defined(USE_SHA1)
460 		case LDNS_DSA:
461 		case LDNS_DSA_NSEC3:
462 			*evp_key = EVP_PKEY_new();
463 			if(!*evp_key) {
464 				log_err("verify: malloc failure in crypto");
465 				return 0;
466 			}
467 			dsa = sldns_key_buf2dsa_raw(key, keylen);
468 			if(!dsa) {
469 				verbose(VERB_QUERY, "verify: "
470 					"sldns_key_buf2dsa_raw failed");
471 				return 0;
472 			}
473 			if(EVP_PKEY_assign_DSA(*evp_key, dsa) == 0) {
474 				verbose(VERB_QUERY, "verify: "
475 					"EVP_PKEY_assign_DSA failed");
476 				return 0;
477 			}
478 #ifdef HAVE_EVP_DSS1
479 			*digest_type = EVP_dss1();
480 #else
481 			*digest_type = EVP_sha1();
482 #endif
483 
484 			break;
485 #endif /* USE_DSA && USE_SHA1 */
486 
487 #if defined(USE_SHA1) || (defined(HAVE_EVP_SHA256) && defined(USE_SHA2)) || (defined(HAVE_EVP_SHA512) && defined(USE_SHA2))
488 #ifdef USE_SHA1
489 		case LDNS_RSASHA1:
490 		case LDNS_RSASHA1_NSEC3:
491 #endif
492 #if defined(HAVE_EVP_SHA256) && defined(USE_SHA2)
493 		case LDNS_RSASHA256:
494 #endif
495 #if defined(HAVE_EVP_SHA512) && defined(USE_SHA2)
496 		case LDNS_RSASHA512:
497 #endif
498 			*evp_key = EVP_PKEY_new();
499 			if(!*evp_key) {
500 				log_err("verify: malloc failure in crypto");
501 				return 0;
502 			}
503 			rsa = sldns_key_buf2rsa_raw(key, keylen);
504 			if(!rsa) {
505 				verbose(VERB_QUERY, "verify: "
506 					"sldns_key_buf2rsa_raw SHA failed");
507 				return 0;
508 			}
509 			if(EVP_PKEY_assign_RSA(*evp_key, rsa) == 0) {
510 				verbose(VERB_QUERY, "verify: "
511 					"EVP_PKEY_assign_RSA SHA failed");
512 				return 0;
513 			}
514 
515 			/* select SHA version */
516 #if defined(HAVE_EVP_SHA256) && defined(USE_SHA2)
517 			if(algo == LDNS_RSASHA256)
518 				*digest_type = EVP_sha256();
519 			else
520 #endif
521 #if defined(HAVE_EVP_SHA512) && defined(USE_SHA2)
522 				if(algo == LDNS_RSASHA512)
523 				*digest_type = EVP_sha512();
524 			else
525 #endif
526 #ifdef USE_SHA1
527 				*digest_type = EVP_sha1();
528 #else
529 				{ verbose(VERB_QUERY, "no digest available"); return 0; }
530 #endif
531 			break;
532 #endif /* defined(USE_SHA1) || (defined(HAVE_EVP_SHA256) && defined(USE_SHA2)) || (defined(HAVE_EVP_SHA512) && defined(USE_SHA2)) */
533 
534 		case LDNS_RSAMD5:
535 			*evp_key = EVP_PKEY_new();
536 			if(!*evp_key) {
537 				log_err("verify: malloc failure in crypto");
538 				return 0;
539 			}
540 			rsa = sldns_key_buf2rsa_raw(key, keylen);
541 			if(!rsa) {
542 				verbose(VERB_QUERY, "verify: "
543 					"sldns_key_buf2rsa_raw MD5 failed");
544 				return 0;
545 			}
546 			if(EVP_PKEY_assign_RSA(*evp_key, rsa) == 0) {
547 				verbose(VERB_QUERY, "verify: "
548 					"EVP_PKEY_assign_RSA MD5 failed");
549 				return 0;
550 			}
551 			*digest_type = EVP_md5();
552 
553 			break;
554 #ifdef USE_GOST
555 		case LDNS_ECC_GOST:
556 			*evp_key = sldns_gost2pkey_raw(key, keylen);
557 			if(!*evp_key) {
558 				verbose(VERB_QUERY, "verify: "
559 					"sldns_gost2pkey_raw failed");
560 				return 0;
561 			}
562 			*digest_type = EVP_get_digestbyname("md_gost94");
563 			if(!*digest_type) {
564 				verbose(VERB_QUERY, "verify: "
565 					"EVP_getdigest md_gost94 failed");
566 				return 0;
567 			}
568 			break;
569 #endif
570 #ifdef USE_ECDSA
571 		case LDNS_ECDSAP256SHA256:
572 			*evp_key = sldns_ecdsa2pkey_raw(key, keylen,
573 				LDNS_ECDSAP256SHA256);
574 			if(!*evp_key) {
575 				verbose(VERB_QUERY, "verify: "
576 					"sldns_ecdsa2pkey_raw failed");
577 				return 0;
578 			}
579 #ifdef USE_ECDSA_EVP_WORKAROUND
580 			*digest_type = &ecdsa_evp_256_md;
581 #else
582 			*digest_type = EVP_sha256();
583 #endif
584 			break;
585 		case LDNS_ECDSAP384SHA384:
586 			*evp_key = sldns_ecdsa2pkey_raw(key, keylen,
587 				LDNS_ECDSAP384SHA384);
588 			if(!*evp_key) {
589 				verbose(VERB_QUERY, "verify: "
590 					"sldns_ecdsa2pkey_raw failed");
591 				return 0;
592 			}
593 #ifdef USE_ECDSA_EVP_WORKAROUND
594 			*digest_type = &ecdsa_evp_384_md;
595 #else
596 			*digest_type = EVP_sha384();
597 #endif
598 			break;
599 #endif /* USE_ECDSA */
600 #ifdef USE_ED25519
601 		case LDNS_ED25519:
602 			*evp_key = sldns_ed255192pkey_raw(key, keylen);
603 			if(!*evp_key) {
604 				verbose(VERB_QUERY, "verify: "
605 					"sldns_ed255192pkey_raw failed");
606 				return 0;
607 			}
608 			*digest_type = NULL;
609 			break;
610 #endif /* USE_ED25519 */
611 #ifdef USE_ED448
612 		case LDNS_ED448:
613 			*evp_key = sldns_ed4482pkey_raw(key, keylen);
614 			if(!*evp_key) {
615 				verbose(VERB_QUERY, "verify: "
616 					"sldns_ed4482pkey_raw failed");
617 				return 0;
618 			}
619 			*digest_type = NULL;
620 			break;
621 #endif /* USE_ED448 */
622 		default:
623 			verbose(VERB_QUERY, "verify: unknown algorithm %d",
624 				algo);
625 			return 0;
626 	}
627 	return 1;
628 }
629 
630 /**
631  * Check a canonical sig+rrset and signature against a dnskey
632  * @param buf: buffer with data to verify, the first rrsig part and the
633  *	canonicalized rrset.
634  * @param algo: DNSKEY algorithm.
635  * @param sigblock: signature rdata field from RRSIG
636  * @param sigblock_len: length of sigblock data.
637  * @param key: public key data from DNSKEY RR.
638  * @param keylen: length of keydata.
639  * @param reason: bogus reason in more detail.
640  * @return secure if verification succeeded, bogus on crypto failure,
641  *	unchecked on format errors and alloc failures.
642  */
643 enum sec_status
644 verify_canonrrset(sldns_buffer* buf, int algo, unsigned char* sigblock,
645 	unsigned int sigblock_len, unsigned char* key, unsigned int keylen,
646 	char** reason)
647 {
648 	const EVP_MD *digest_type;
649 	EVP_MD_CTX* ctx;
650 	int res, dofree = 0, docrypto_free = 0;
651 	EVP_PKEY *evp_key = NULL;
652 
653 #ifndef USE_DSA
654 	if((algo == LDNS_DSA || algo == LDNS_DSA_NSEC3) &&(fake_dsa||fake_sha1))
655 		return sec_status_secure;
656 #endif
657 #ifndef USE_SHA1
658 	if(fake_sha1 && (algo == LDNS_DSA || algo == LDNS_DSA_NSEC3 || algo == LDNS_RSASHA1 || algo == LDNS_RSASHA1_NSEC3))
659 		return sec_status_secure;
660 #endif
661 
662 	if(!setup_key_digest(algo, &evp_key, &digest_type, key, keylen)) {
663 		verbose(VERB_QUERY, "verify: failed to setup key");
664 		*reason = "use of key for crypto failed";
665 		EVP_PKEY_free(evp_key);
666 		return sec_status_bogus;
667 	}
668 #ifdef USE_DSA
669 	/* if it is a DSA signature in bind format, convert to DER format */
670 	if((algo == LDNS_DSA || algo == LDNS_DSA_NSEC3) &&
671 		sigblock_len == 1+2*SHA_DIGEST_LENGTH) {
672 		if(!setup_dsa_sig(&sigblock, &sigblock_len)) {
673 			verbose(VERB_QUERY, "verify: failed to setup DSA sig");
674 			*reason = "use of key for DSA crypto failed";
675 			EVP_PKEY_free(evp_key);
676 			return sec_status_bogus;
677 		}
678 		docrypto_free = 1;
679 	}
680 #endif
681 #if defined(USE_ECDSA) && defined(USE_DSA)
682 	else
683 #endif
684 #ifdef USE_ECDSA
685 	if(algo == LDNS_ECDSAP256SHA256 || algo == LDNS_ECDSAP384SHA384) {
686 		/* EVP uses ASN prefix on sig, which is not in the wire data */
687 		if(!setup_ecdsa_sig(&sigblock, &sigblock_len)) {
688 			verbose(VERB_QUERY, "verify: failed to setup ECDSA sig");
689 			*reason = "use of signature for ECDSA crypto failed";
690 			EVP_PKEY_free(evp_key);
691 			return sec_status_bogus;
692 		}
693 		dofree = 1;
694 	}
695 #endif /* USE_ECDSA */
696 
697 	/* do the signature cryptography work */
698 #ifdef HAVE_EVP_MD_CTX_NEW
699 	ctx = EVP_MD_CTX_new();
700 #else
701 	ctx = (EVP_MD_CTX*)malloc(sizeof(*ctx));
702 	if(ctx) EVP_MD_CTX_init(ctx);
703 #endif
704 	if(!ctx) {
705 		log_err("EVP_MD_CTX_new: malloc failure");
706 		EVP_PKEY_free(evp_key);
707 		if(dofree) free(sigblock);
708 		else if(docrypto_free) OPENSSL_free(sigblock);
709 		return sec_status_unchecked;
710 	}
711 #ifndef HAVE_EVP_DIGESTVERIFY
712 	if(EVP_DigestInit(ctx, digest_type) == 0) {
713 		verbose(VERB_QUERY, "verify: EVP_DigestInit failed");
714 #ifdef HAVE_EVP_MD_CTX_NEW
715 		EVP_MD_CTX_destroy(ctx);
716 #else
717 		EVP_MD_CTX_cleanup(ctx);
718 		free(ctx);
719 #endif
720 		EVP_PKEY_free(evp_key);
721 		if(dofree) free(sigblock);
722 		else if(docrypto_free) OPENSSL_free(sigblock);
723 		return sec_status_unchecked;
724 	}
725 	if(EVP_DigestUpdate(ctx, (unsigned char*)sldns_buffer_begin(buf),
726 		(unsigned int)sldns_buffer_limit(buf)) == 0) {
727 		verbose(VERB_QUERY, "verify: EVP_DigestUpdate failed");
728 #ifdef HAVE_EVP_MD_CTX_NEW
729 		EVP_MD_CTX_destroy(ctx);
730 #else
731 		EVP_MD_CTX_cleanup(ctx);
732 		free(ctx);
733 #endif
734 		EVP_PKEY_free(evp_key);
735 		if(dofree) free(sigblock);
736 		else if(docrypto_free) OPENSSL_free(sigblock);
737 		return sec_status_unchecked;
738 	}
739 
740 	res = EVP_VerifyFinal(ctx, sigblock, sigblock_len, evp_key);
741 #else /* HAVE_EVP_DIGESTVERIFY */
742 	if(EVP_DigestVerifyInit(ctx, NULL, digest_type, NULL, evp_key) == 0) {
743 		verbose(VERB_QUERY, "verify: EVP_DigestVerifyInit failed");
744 #ifdef HAVE_EVP_MD_CTX_NEW
745 		EVP_MD_CTX_destroy(ctx);
746 #else
747 		EVP_MD_CTX_cleanup(ctx);
748 		free(ctx);
749 #endif
750 		EVP_PKEY_free(evp_key);
751 		if(dofree) free(sigblock);
752 		else if(docrypto_free) OPENSSL_free(sigblock);
753 		return sec_status_unchecked;
754 	}
755 	res = EVP_DigestVerify(ctx, sigblock, sigblock_len,
756 		(unsigned char*)sldns_buffer_begin(buf),
757 		sldns_buffer_limit(buf));
758 #endif
759 #ifdef HAVE_EVP_MD_CTX_NEW
760 	EVP_MD_CTX_destroy(ctx);
761 #else
762 	EVP_MD_CTX_cleanup(ctx);
763 	free(ctx);
764 #endif
765 	EVP_PKEY_free(evp_key);
766 
767 	if(dofree) free(sigblock);
768 	else if(docrypto_free) OPENSSL_free(sigblock);
769 
770 	if(res == 1) {
771 		return sec_status_secure;
772 	} else if(res == 0) {
773 		verbose(VERB_QUERY, "verify: signature mismatch");
774 		*reason = "signature crypto failed";
775 		return sec_status_bogus;
776 	}
777 
778 	log_crypto_error("verify:", ERR_get_error());
779 	return sec_status_unchecked;
780 }
781 
782 /**************************************************/
783 #elif defined(HAVE_NSS)
784 /* libnss implementation */
785 /* nss3 */
786 #include "sechash.h"
787 #include "pk11pub.h"
788 #include "keyhi.h"
789 #include "secerr.h"
790 #include "cryptohi.h"
791 /* nspr4 */
792 #include "prerror.h"
793 
794 /* return size of digest if supported, or 0 otherwise */
795 size_t
796 nsec3_hash_algo_size_supported(int id)
797 {
798 	switch(id) {
799 	case NSEC3_HASH_SHA1:
800 		return SHA1_LENGTH;
801 	default:
802 		return 0;
803 	}
804 }
805 
806 /* perform nsec3 hash. return false on failure */
807 int
808 secalgo_nsec3_hash(int algo, unsigned char* buf, size_t len,
809         unsigned char* res)
810 {
811 	switch(algo) {
812 	case NSEC3_HASH_SHA1:
813 		(void)HASH_HashBuf(HASH_AlgSHA1, res, buf, (unsigned long)len);
814 		return 1;
815 	default:
816 		return 0;
817 	}
818 }
819 
820 void
821 secalgo_hash_sha256(unsigned char* buf, size_t len, unsigned char* res)
822 {
823 	(void)HASH_HashBuf(HASH_AlgSHA256, res, buf, (unsigned long)len);
824 }
825 
826 size_t
827 ds_digest_size_supported(int algo)
828 {
829 	/* uses libNSS */
830 	switch(algo) {
831 #ifdef USE_SHA1
832 		case LDNS_SHA1:
833 			return SHA1_LENGTH;
834 #endif
835 #ifdef USE_SHA2
836 		case LDNS_SHA256:
837 			return SHA256_LENGTH;
838 #endif
839 #ifdef USE_ECDSA
840 		case LDNS_SHA384:
841 			return SHA384_LENGTH;
842 #endif
843 		/* GOST not supported in NSS */
844 		case LDNS_HASH_GOST:
845 		default: break;
846 	}
847 	return 0;
848 }
849 
850 int
851 secalgo_ds_digest(int algo, unsigned char* buf, size_t len,
852 	unsigned char* res)
853 {
854 	/* uses libNSS */
855 	switch(algo) {
856 #ifdef USE_SHA1
857 		case LDNS_SHA1:
858 			return HASH_HashBuf(HASH_AlgSHA1, res, buf, len)
859 				== SECSuccess;
860 #endif
861 #if defined(USE_SHA2)
862 		case LDNS_SHA256:
863 			return HASH_HashBuf(HASH_AlgSHA256, res, buf, len)
864 				== SECSuccess;
865 #endif
866 #ifdef USE_ECDSA
867 		case LDNS_SHA384:
868 			return HASH_HashBuf(HASH_AlgSHA384, res, buf, len)
869 				== SECSuccess;
870 #endif
871 		case LDNS_HASH_GOST:
872 		default:
873 			verbose(VERB_QUERY, "unknown DS digest algorithm %d",
874 				algo);
875 			break;
876 	}
877 	return 0;
878 }
879 
880 int
881 dnskey_algo_id_is_supported(int id)
882 {
883 	/* uses libNSS */
884 	switch(id) {
885 	case LDNS_RSAMD5:
886 		/* RFC 6725 deprecates RSAMD5 */
887 		return 0;
888 #if defined(USE_SHA1) || defined(USE_SHA2)
889 #if defined(USE_DSA) && defined(USE_SHA1)
890 	case LDNS_DSA:
891 	case LDNS_DSA_NSEC3:
892 #endif
893 #ifdef USE_SHA1
894 	case LDNS_RSASHA1:
895 	case LDNS_RSASHA1_NSEC3:
896 #endif
897 #ifdef USE_SHA2
898 	case LDNS_RSASHA256:
899 #endif
900 #ifdef USE_SHA2
901 	case LDNS_RSASHA512:
902 #endif
903 		return 1;
904 #endif /* SHA1 or SHA2 */
905 
906 #ifdef USE_ECDSA
907 	case LDNS_ECDSAP256SHA256:
908 	case LDNS_ECDSAP384SHA384:
909 		return PK11_TokenExists(CKM_ECDSA);
910 #endif
911 	case LDNS_ECC_GOST:
912 	default:
913 		return 0;
914 	}
915 }
916 
917 /* return a new public key for NSS */
918 static SECKEYPublicKey* nss_key_create(KeyType ktype)
919 {
920 	SECKEYPublicKey* key;
921 	PLArenaPool* arena = PORT_NewArena(DER_DEFAULT_CHUNKSIZE);
922 	if(!arena) {
923 		log_err("out of memory, PORT_NewArena failed");
924 		return NULL;
925 	}
926 	key = PORT_ArenaZNew(arena, SECKEYPublicKey);
927 	if(!key) {
928 		log_err("out of memory, PORT_ArenaZNew failed");
929 		PORT_FreeArena(arena, PR_FALSE);
930 		return NULL;
931 	}
932 	key->arena = arena;
933 	key->keyType = ktype;
934 	key->pkcs11Slot = NULL;
935 	key->pkcs11ID = CK_INVALID_HANDLE;
936 	return key;
937 }
938 
939 static SECKEYPublicKey* nss_buf2ecdsa(unsigned char* key, size_t len, int algo)
940 {
941 	SECKEYPublicKey* pk;
942 	SECItem pub = {siBuffer, NULL, 0};
943 	SECItem params = {siBuffer, NULL, 0};
944 	static unsigned char param256[] = {
945 		/* OBJECTIDENTIFIER 1.2.840.10045.3.1.7 (P-256)
946 		 * {iso(1) member-body(2) us(840) ansi-x962(10045) curves(3) prime(1) prime256v1(7)} */
947 		0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, 0x01, 0x07
948 	};
949 	static unsigned char param384[] = {
950 		/* OBJECTIDENTIFIER 1.3.132.0.34 (P-384)
951 		 * {iso(1) identified-organization(3) certicom(132) curve(0) ansip384r1(34)} */
952 		0x06, 0x05, 0x2b, 0x81, 0x04, 0x00, 0x22
953 	};
954 	unsigned char buf[256+2]; /* sufficient for 2*384/8+1 */
955 
956 	/* check length, which uncompressed must be 2 bignums */
957 	if(algo == LDNS_ECDSAP256SHA256) {
958 		if(len != 2*256/8) return NULL;
959 		/* ECCurve_X9_62_PRIME_256V1 */
960 	} else if(algo == LDNS_ECDSAP384SHA384) {
961 		if(len != 2*384/8) return NULL;
962 		/* ECCurve_X9_62_PRIME_384R1 */
963 	} else    return NULL;
964 
965 	buf[0] = 0x04; /* POINT_FORM_UNCOMPRESSED */
966 	memmove(buf+1, key, len);
967 	pub.data = buf;
968 	pub.len = len+1;
969 	if(algo == LDNS_ECDSAP256SHA256) {
970 		params.data = param256;
971 		params.len = sizeof(param256);
972 	} else {
973 		params.data = param384;
974 		params.len = sizeof(param384);
975 	}
976 
977 	pk = nss_key_create(ecKey);
978 	if(!pk)
979 		return NULL;
980 	pk->u.ec.size = (len/2)*8;
981 	if(SECITEM_CopyItem(pk->arena, &pk->u.ec.publicValue, &pub)) {
982 		SECKEY_DestroyPublicKey(pk);
983 		return NULL;
984 	}
985 	if(SECITEM_CopyItem(pk->arena, &pk->u.ec.DEREncodedParams, &params)) {
986 		SECKEY_DestroyPublicKey(pk);
987 		return NULL;
988 	}
989 
990 	return pk;
991 }
992 
993 #if defined(USE_DSA) && defined(USE_SHA1)
994 static SECKEYPublicKey* nss_buf2dsa(unsigned char* key, size_t len)
995 {
996 	SECKEYPublicKey* pk;
997 	uint8_t T;
998 	uint16_t length;
999 	uint16_t offset;
1000 	SECItem Q = {siBuffer, NULL, 0};
1001 	SECItem P = {siBuffer, NULL, 0};
1002 	SECItem G = {siBuffer, NULL, 0};
1003 	SECItem Y = {siBuffer, NULL, 0};
1004 
1005 	if(len == 0)
1006 		return NULL;
1007 	T = (uint8_t)key[0];
1008 	length = (64 + T * 8);
1009 	offset = 1;
1010 
1011 	if (T > 8) {
1012 		return NULL;
1013 	}
1014 	if(len < (size_t)1 + SHA1_LENGTH + 3*length)
1015 		return NULL;
1016 
1017 	Q.data = key+offset;
1018 	Q.len = SHA1_LENGTH;
1019 	offset += SHA1_LENGTH;
1020 
1021 	P.data = key+offset;
1022 	P.len = length;
1023 	offset += length;
1024 
1025 	G.data = key+offset;
1026 	G.len = length;
1027 	offset += length;
1028 
1029 	Y.data = key+offset;
1030 	Y.len = length;
1031 	offset += length;
1032 
1033 	pk = nss_key_create(dsaKey);
1034 	if(!pk)
1035 		return NULL;
1036 	if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.params.prime, &P)) {
1037 		SECKEY_DestroyPublicKey(pk);
1038 		return NULL;
1039 	}
1040 	if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.params.subPrime, &Q)) {
1041 		SECKEY_DestroyPublicKey(pk);
1042 		return NULL;
1043 	}
1044 	if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.params.base, &G)) {
1045 		SECKEY_DestroyPublicKey(pk);
1046 		return NULL;
1047 	}
1048 	if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.publicValue, &Y)) {
1049 		SECKEY_DestroyPublicKey(pk);
1050 		return NULL;
1051 	}
1052 	return pk;
1053 }
1054 #endif /* USE_DSA && USE_SHA1 */
1055 
1056 static SECKEYPublicKey* nss_buf2rsa(unsigned char* key, size_t len)
1057 {
1058 	SECKEYPublicKey* pk;
1059 	uint16_t exp;
1060 	uint16_t offset;
1061 	uint16_t int16;
1062 	SECItem modulus = {siBuffer, NULL, 0};
1063 	SECItem exponent = {siBuffer, NULL, 0};
1064 	if(len == 0)
1065 		return NULL;
1066 	if(key[0] == 0) {
1067 		if(len < 3)
1068 			return NULL;
1069 		/* the exponent is too large so it's places further */
1070 		memmove(&int16, key+1, 2);
1071 		exp = ntohs(int16);
1072 		offset = 3;
1073 	} else {
1074 		exp = key[0];
1075 		offset = 1;
1076 	}
1077 
1078 	/* key length at least one */
1079 	if(len < (size_t)offset + exp + 1)
1080 		return NULL;
1081 
1082 	exponent.data = key+offset;
1083 	exponent.len = exp;
1084 	offset += exp;
1085 	modulus.data = key+offset;
1086 	modulus.len = (len - offset);
1087 
1088 	pk = nss_key_create(rsaKey);
1089 	if(!pk)
1090 		return NULL;
1091 	if(SECITEM_CopyItem(pk->arena, &pk->u.rsa.modulus, &modulus)) {
1092 		SECKEY_DestroyPublicKey(pk);
1093 		return NULL;
1094 	}
1095 	if(SECITEM_CopyItem(pk->arena, &pk->u.rsa.publicExponent, &exponent)) {
1096 		SECKEY_DestroyPublicKey(pk);
1097 		return NULL;
1098 	}
1099 	return pk;
1100 }
1101 
1102 /**
1103  * Setup key and digest for verification. Adjust sig if necessary.
1104  *
1105  * @param algo: key algorithm
1106  * @param evp_key: EVP PKEY public key to create.
1107  * @param digest_type: digest type to use
1108  * @param key: key to setup for.
1109  * @param keylen: length of key.
1110  * @param prefix: if returned, the ASN prefix for the hashblob.
1111  * @param prefixlen: length of the prefix.
1112  * @return false on failure.
1113  */
1114 static int
1115 nss_setup_key_digest(int algo, SECKEYPublicKey** pubkey, HASH_HashType* htype,
1116 	unsigned char* key, size_t keylen, unsigned char** prefix,
1117 	size_t* prefixlen)
1118 {
1119 	/* uses libNSS */
1120 
1121 	/* hash prefix for md5, RFC2537 */
1122 	static unsigned char p_md5[] = {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a,
1123 	0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10};
1124 	/* hash prefix to prepend to hash output, from RFC3110 */
1125 	static unsigned char p_sha1[] = {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2B,
1126 		0x0E, 0x03, 0x02, 0x1A, 0x05, 0x00, 0x04, 0x14};
1127 	/* from RFC5702 */
1128 	static unsigned char p_sha256[] = {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60,
1129 	0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
1130 	static unsigned char p_sha512[] = {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60,
1131 	0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
1132 	/* from RFC6234 */
1133 	/* for future RSASHA384 ..
1134 	static unsigned char p_sha384[] = {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60,
1135 	0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30};
1136 	*/
1137 
1138 	switch(algo) {
1139 
1140 #if defined(USE_SHA1) || defined(USE_SHA2)
1141 #if defined(USE_DSA) && defined(USE_SHA1)
1142 		case LDNS_DSA:
1143 		case LDNS_DSA_NSEC3:
1144 			*pubkey = nss_buf2dsa(key, keylen);
1145 			if(!*pubkey) {
1146 				log_err("verify: malloc failure in crypto");
1147 				return 0;
1148 			}
1149 			*htype = HASH_AlgSHA1;
1150 			/* no prefix for DSA verification */
1151 			break;
1152 #endif
1153 #ifdef USE_SHA1
1154 		case LDNS_RSASHA1:
1155 		case LDNS_RSASHA1_NSEC3:
1156 #endif
1157 #ifdef USE_SHA2
1158 		case LDNS_RSASHA256:
1159 #endif
1160 #ifdef USE_SHA2
1161 		case LDNS_RSASHA512:
1162 #endif
1163 			*pubkey = nss_buf2rsa(key, keylen);
1164 			if(!*pubkey) {
1165 				log_err("verify: malloc failure in crypto");
1166 				return 0;
1167 			}
1168 			/* select SHA version */
1169 #ifdef USE_SHA2
1170 			if(algo == LDNS_RSASHA256) {
1171 				*htype = HASH_AlgSHA256;
1172 				*prefix = p_sha256;
1173 				*prefixlen = sizeof(p_sha256);
1174 			} else
1175 #endif
1176 #ifdef USE_SHA2
1177 				if(algo == LDNS_RSASHA512) {
1178 				*htype = HASH_AlgSHA512;
1179 				*prefix = p_sha512;
1180 				*prefixlen = sizeof(p_sha512);
1181 			} else
1182 #endif
1183 #ifdef USE_SHA1
1184 			{
1185 				*htype = HASH_AlgSHA1;
1186 				*prefix = p_sha1;
1187 				*prefixlen = sizeof(p_sha1);
1188 			}
1189 #else
1190 			{
1191 				verbose(VERB_QUERY, "verify: no digest algo");
1192 				return 0;
1193 			}
1194 #endif
1195 
1196 			break;
1197 #endif /* SHA1 or SHA2 */
1198 
1199 		case LDNS_RSAMD5:
1200 			*pubkey = nss_buf2rsa(key, keylen);
1201 			if(!*pubkey) {
1202 				log_err("verify: malloc failure in crypto");
1203 				return 0;
1204 			}
1205 			*htype = HASH_AlgMD5;
1206 			*prefix = p_md5;
1207 			*prefixlen = sizeof(p_md5);
1208 
1209 			break;
1210 #ifdef USE_ECDSA
1211 		case LDNS_ECDSAP256SHA256:
1212 			*pubkey = nss_buf2ecdsa(key, keylen,
1213 				LDNS_ECDSAP256SHA256);
1214 			if(!*pubkey) {
1215 				log_err("verify: malloc failure in crypto");
1216 				return 0;
1217 			}
1218 			*htype = HASH_AlgSHA256;
1219 			/* no prefix for DSA verification */
1220 			break;
1221 		case LDNS_ECDSAP384SHA384:
1222 			*pubkey = nss_buf2ecdsa(key, keylen,
1223 				LDNS_ECDSAP384SHA384);
1224 			if(!*pubkey) {
1225 				log_err("verify: malloc failure in crypto");
1226 				return 0;
1227 			}
1228 			*htype = HASH_AlgSHA384;
1229 			/* no prefix for DSA verification */
1230 			break;
1231 #endif /* USE_ECDSA */
1232 		case LDNS_ECC_GOST:
1233 		default:
1234 			verbose(VERB_QUERY, "verify: unknown algorithm %d",
1235 				algo);
1236 			return 0;
1237 	}
1238 	return 1;
1239 }
1240 
1241 /**
1242  * Check a canonical sig+rrset and signature against a dnskey
1243  * @param buf: buffer with data to verify, the first rrsig part and the
1244  *	canonicalized rrset.
1245  * @param algo: DNSKEY algorithm.
1246  * @param sigblock: signature rdata field from RRSIG
1247  * @param sigblock_len: length of sigblock data.
1248  * @param key: public key data from DNSKEY RR.
1249  * @param keylen: length of keydata.
1250  * @param reason: bogus reason in more detail.
1251  * @return secure if verification succeeded, bogus on crypto failure,
1252  *	unchecked on format errors and alloc failures.
1253  */
1254 enum sec_status
1255 verify_canonrrset(sldns_buffer* buf, int algo, unsigned char* sigblock,
1256 	unsigned int sigblock_len, unsigned char* key, unsigned int keylen,
1257 	char** reason)
1258 {
1259 	/* uses libNSS */
1260 	/* large enough for the different hashes */
1261 	unsigned char hash[HASH_LENGTH_MAX];
1262 	unsigned char hash2[HASH_LENGTH_MAX*2];
1263 	HASH_HashType htype = 0;
1264 	SECKEYPublicKey* pubkey = NULL;
1265 	SECItem secsig = {siBuffer, sigblock, sigblock_len};
1266 	SECItem sechash = {siBuffer, hash, 0};
1267 	SECStatus res;
1268 	unsigned char* prefix = NULL; /* prefix for hash, RFC3110, RFC5702 */
1269 	size_t prefixlen = 0;
1270 	int err;
1271 
1272 	if(!nss_setup_key_digest(algo, &pubkey, &htype, key, keylen,
1273 		&prefix, &prefixlen)) {
1274 		verbose(VERB_QUERY, "verify: failed to setup key");
1275 		*reason = "use of key for crypto failed";
1276 		SECKEY_DestroyPublicKey(pubkey);
1277 		return sec_status_bogus;
1278 	}
1279 
1280 #if defined(USE_DSA) && defined(USE_SHA1)
1281 	/* need to convert DSA, ECDSA signatures? */
1282 	if((algo == LDNS_DSA || algo == LDNS_DSA_NSEC3)) {
1283 		if(sigblock_len == 1+2*SHA1_LENGTH) {
1284 			secsig.data ++;
1285 			secsig.len --;
1286 		} else {
1287 			SECItem* p = DSAU_DecodeDerSig(&secsig);
1288 			if(!p) {
1289 				verbose(VERB_QUERY, "verify: failed DER decode");
1290 				*reason = "signature DER decode failed";
1291 				SECKEY_DestroyPublicKey(pubkey);
1292 				return sec_status_bogus;
1293 			}
1294 			if(SECITEM_CopyItem(pubkey->arena, &secsig, p)) {
1295 				log_err("alloc failure in DER decode");
1296 				SECKEY_DestroyPublicKey(pubkey);
1297 				SECITEM_FreeItem(p, PR_TRUE);
1298 				return sec_status_unchecked;
1299 			}
1300 			SECITEM_FreeItem(p, PR_TRUE);
1301 		}
1302 	}
1303 #endif /* USE_DSA */
1304 
1305 	/* do the signature cryptography work */
1306 	/* hash the data */
1307 	sechash.len = HASH_ResultLen(htype);
1308 	if(sechash.len > sizeof(hash)) {
1309 		verbose(VERB_QUERY, "verify: hash too large for buffer");
1310 		SECKEY_DestroyPublicKey(pubkey);
1311 		return sec_status_unchecked;
1312 	}
1313 	if(HASH_HashBuf(htype, hash, (unsigned char*)sldns_buffer_begin(buf),
1314 		(unsigned int)sldns_buffer_limit(buf)) != SECSuccess) {
1315 		verbose(VERB_QUERY, "verify: HASH_HashBuf failed");
1316 		SECKEY_DestroyPublicKey(pubkey);
1317 		return sec_status_unchecked;
1318 	}
1319 	if(prefix) {
1320 		int hashlen = sechash.len;
1321 		if(prefixlen+hashlen > sizeof(hash2)) {
1322 			verbose(VERB_QUERY, "verify: hashprefix too large");
1323 			SECKEY_DestroyPublicKey(pubkey);
1324 			return sec_status_unchecked;
1325 		}
1326 		sechash.data = hash2;
1327 		sechash.len = prefixlen+hashlen;
1328 		memcpy(sechash.data, prefix, prefixlen);
1329 		memmove(sechash.data+prefixlen, hash, hashlen);
1330 	}
1331 
1332 	/* verify the signature */
1333 	res = PK11_Verify(pubkey, &secsig, &sechash, NULL /*wincx*/);
1334 	SECKEY_DestroyPublicKey(pubkey);
1335 
1336 	if(res == SECSuccess) {
1337 		return sec_status_secure;
1338 	}
1339 	err = PORT_GetError();
1340 	if(err != SEC_ERROR_BAD_SIGNATURE) {
1341 		/* failed to verify */
1342 		verbose(VERB_QUERY, "verify: PK11_Verify failed: %s",
1343 			PORT_ErrorToString(err));
1344 		/* if it is not supported, like ECC is removed, we get,
1345 		 * SEC_ERROR_NO_MODULE */
1346 		if(err == SEC_ERROR_NO_MODULE)
1347 			return sec_status_unchecked;
1348 		/* but other errors are commonly returned
1349 		 * for a bad signature from NSS.  Thus we return bogus,
1350 		 * not unchecked */
1351 		*reason = "signature crypto failed";
1352 		return sec_status_bogus;
1353 	}
1354 	verbose(VERB_QUERY, "verify: signature mismatch: %s",
1355 		PORT_ErrorToString(err));
1356 	*reason = "signature crypto failed";
1357 	return sec_status_bogus;
1358 }
1359 
1360 #elif defined(HAVE_NETTLE)
1361 
1362 #include "sha.h"
1363 #include "bignum.h"
1364 #include "macros.h"
1365 #include "rsa.h"
1366 #include "dsa.h"
1367 #ifdef HAVE_NETTLE_DSA_COMPAT_H
1368 #include "dsa-compat.h"
1369 #endif
1370 #include "asn1.h"
1371 #ifdef USE_ECDSA
1372 #include "ecdsa.h"
1373 #include "ecc-curve.h"
1374 #endif
1375 #ifdef HAVE_NETTLE_EDDSA_H
1376 #include "eddsa.h"
1377 #endif
1378 
1379 static int
1380 _digest_nettle(int algo, uint8_t* buf, size_t len,
1381 	unsigned char* res)
1382 {
1383 	switch(algo) {
1384 		case SHA1_DIGEST_SIZE:
1385 		{
1386 			struct sha1_ctx ctx;
1387 			sha1_init(&ctx);
1388 			sha1_update(&ctx, len, buf);
1389 			sha1_digest(&ctx, SHA1_DIGEST_SIZE, res);
1390 			return 1;
1391 		}
1392 		case SHA256_DIGEST_SIZE:
1393 		{
1394 			struct sha256_ctx ctx;
1395 			sha256_init(&ctx);
1396 			sha256_update(&ctx, len, buf);
1397 			sha256_digest(&ctx, SHA256_DIGEST_SIZE, res);
1398 			return 1;
1399 		}
1400 		case SHA384_DIGEST_SIZE:
1401 		{
1402 			struct sha384_ctx ctx;
1403 			sha384_init(&ctx);
1404 			sha384_update(&ctx, len, buf);
1405 			sha384_digest(&ctx, SHA384_DIGEST_SIZE, res);
1406 			return 1;
1407 		}
1408 		case SHA512_DIGEST_SIZE:
1409 		{
1410 			struct sha512_ctx ctx;
1411 			sha512_init(&ctx);
1412 			sha512_update(&ctx, len, buf);
1413 			sha512_digest(&ctx, SHA512_DIGEST_SIZE, res);
1414 			return 1;
1415 		}
1416 		default:
1417 			break;
1418 	}
1419 	return 0;
1420 }
1421 
1422 /* return size of digest if supported, or 0 otherwise */
1423 size_t
1424 nsec3_hash_algo_size_supported(int id)
1425 {
1426 	switch(id) {
1427 	case NSEC3_HASH_SHA1:
1428 		return SHA1_DIGEST_SIZE;
1429 	default:
1430 		return 0;
1431 	}
1432 }
1433 
1434 /* perform nsec3 hash. return false on failure */
1435 int
1436 secalgo_nsec3_hash(int algo, unsigned char* buf, size_t len,
1437         unsigned char* res)
1438 {
1439 	switch(algo) {
1440 	case NSEC3_HASH_SHA1:
1441 		return _digest_nettle(SHA1_DIGEST_SIZE, (uint8_t*)buf, len,
1442 			res);
1443 	default:
1444 		return 0;
1445 	}
1446 }
1447 
1448 void
1449 secalgo_hash_sha256(unsigned char* buf, size_t len, unsigned char* res)
1450 {
1451 	_digest_nettle(SHA256_DIGEST_SIZE, (uint8_t*)buf, len, res);
1452 }
1453 
1454 /**
1455  * Return size of DS digest according to its hash algorithm.
1456  * @param algo: DS digest algo.
1457  * @return size in bytes of digest, or 0 if not supported.
1458  */
1459 size_t
1460 ds_digest_size_supported(int algo)
1461 {
1462 	switch(algo) {
1463 		case LDNS_SHA1:
1464 #ifdef USE_SHA1
1465 			return SHA1_DIGEST_SIZE;
1466 #else
1467 			if(fake_sha1) return 20;
1468 			return 0;
1469 #endif
1470 #ifdef USE_SHA2
1471 		case LDNS_SHA256:
1472 			return SHA256_DIGEST_SIZE;
1473 #endif
1474 #ifdef USE_ECDSA
1475 		case LDNS_SHA384:
1476 			return SHA384_DIGEST_SIZE;
1477 #endif
1478 		/* GOST not supported */
1479 		case LDNS_HASH_GOST:
1480 		default:
1481 			break;
1482 	}
1483 	return 0;
1484 }
1485 
1486 int
1487 secalgo_ds_digest(int algo, unsigned char* buf, size_t len,
1488 	unsigned char* res)
1489 {
1490 	switch(algo) {
1491 #ifdef USE_SHA1
1492 		case LDNS_SHA1:
1493 			return _digest_nettle(SHA1_DIGEST_SIZE, buf, len, res);
1494 #endif
1495 #if defined(USE_SHA2)
1496 		case LDNS_SHA256:
1497 			return _digest_nettle(SHA256_DIGEST_SIZE, buf, len, res);
1498 #endif
1499 #ifdef USE_ECDSA
1500 		case LDNS_SHA384:
1501 			return _digest_nettle(SHA384_DIGEST_SIZE, buf, len, res);
1502 
1503 #endif
1504 		case LDNS_HASH_GOST:
1505 		default:
1506 			verbose(VERB_QUERY, "unknown DS digest algorithm %d",
1507 				algo);
1508 			break;
1509 	}
1510 	return 0;
1511 }
1512 
1513 int
1514 dnskey_algo_id_is_supported(int id)
1515 {
1516 	/* uses libnettle */
1517 	switch(id) {
1518 	case LDNS_DSA:
1519 	case LDNS_DSA_NSEC3:
1520 #if defined(USE_DSA) && defined(USE_SHA1)
1521 		return 1;
1522 #else
1523 		if(fake_dsa || fake_sha1) return 1;
1524 		return 0;
1525 #endif
1526 	case LDNS_RSASHA1:
1527 	case LDNS_RSASHA1_NSEC3:
1528 #ifdef USE_SHA1
1529 		return 1;
1530 #else
1531 		if(fake_sha1) return 1;
1532 		return 0;
1533 #endif
1534 #ifdef USE_SHA2
1535 	case LDNS_RSASHA256:
1536 	case LDNS_RSASHA512:
1537 #endif
1538 #ifdef USE_ECDSA
1539 	case LDNS_ECDSAP256SHA256:
1540 	case LDNS_ECDSAP384SHA384:
1541 #endif
1542 		return 1;
1543 #ifdef USE_ED25519
1544 	case LDNS_ED25519:
1545 		return 1;
1546 #endif
1547 	case LDNS_RSAMD5: /* RFC 6725 deprecates RSAMD5 */
1548 	case LDNS_ECC_GOST:
1549 	default:
1550 		return 0;
1551 	}
1552 }
1553 
1554 #if defined(USE_DSA) && defined(USE_SHA1)
1555 static char *
1556 _verify_nettle_dsa(sldns_buffer* buf, unsigned char* sigblock,
1557 	unsigned int sigblock_len, unsigned char* key, unsigned int keylen)
1558 {
1559 	uint8_t digest[SHA1_DIGEST_SIZE];
1560 	uint8_t key_t_value;
1561 	int res = 0;
1562 	size_t offset;
1563 	struct dsa_public_key pubkey;
1564 	struct dsa_signature signature;
1565 	unsigned int expected_len;
1566 
1567 	/* Extract DSA signature from the record */
1568 	nettle_dsa_signature_init(&signature);
1569 	/* Signature length: 41 bytes - RFC 2536 sec. 3 */
1570 	if(sigblock_len == 41) {
1571 		if(key[0] != sigblock[0])
1572 			return "invalid T value in DSA signature or pubkey";
1573 		nettle_mpz_set_str_256_u(signature.r, 20, sigblock+1);
1574 		nettle_mpz_set_str_256_u(signature.s, 20, sigblock+1+20);
1575 	} else {
1576 		/* DER encoded, decode the ASN1 notated R and S bignums */
1577 		/* SEQUENCE { r INTEGER, s INTEGER } */
1578 		struct asn1_der_iterator i, seq;
1579 		if(asn1_der_iterator_first(&i, sigblock_len,
1580 			(uint8_t*)sigblock) != ASN1_ITERATOR_CONSTRUCTED
1581 			|| i.type != ASN1_SEQUENCE)
1582 			return "malformed DER encoded DSA signature";
1583 		/* decode this element of i using the seq iterator */
1584 		if(asn1_der_decode_constructed(&i, &seq) !=
1585 			ASN1_ITERATOR_PRIMITIVE || seq.type != ASN1_INTEGER)
1586 			return "malformed DER encoded DSA signature";
1587 		if(!asn1_der_get_bignum(&seq, signature.r, 20*8))
1588 			return "malformed DER encoded DSA signature";
1589 		if(asn1_der_iterator_next(&seq) != ASN1_ITERATOR_PRIMITIVE
1590 			|| seq.type != ASN1_INTEGER)
1591 			return "malformed DER encoded DSA signature";
1592 		if(!asn1_der_get_bignum(&seq, signature.s, 20*8))
1593 			return "malformed DER encoded DSA signature";
1594 		if(asn1_der_iterator_next(&i) != ASN1_ITERATOR_END)
1595 			return "malformed DER encoded DSA signature";
1596 	}
1597 
1598 	/* Validate T values constraints - RFC 2536 sec. 2 & sec. 3 */
1599 	key_t_value = key[0];
1600 	if (key_t_value > 8) {
1601 		return "invalid T value in DSA pubkey";
1602 	}
1603 
1604 	/* Pubkey minimum length: 21 bytes - RFC 2536 sec. 2 */
1605 	if (keylen < 21) {
1606 		return "DSA pubkey too short";
1607 	}
1608 
1609 	expected_len =   1 +		/* T */
1610 		        20 +		/* Q */
1611 		       (64 + key_t_value*8) +	/* P */
1612 		       (64 + key_t_value*8) +	/* G */
1613 		       (64 + key_t_value*8);	/* Y */
1614 	if (keylen != expected_len ) {
1615 		return "invalid DSA pubkey length";
1616 	}
1617 
1618 	/* Extract DSA pubkey from the record */
1619 	nettle_dsa_public_key_init(&pubkey);
1620 	offset = 1;
1621 	nettle_mpz_set_str_256_u(pubkey.q, 20, key+offset);
1622 	offset += 20;
1623 	nettle_mpz_set_str_256_u(pubkey.p, (64 + key_t_value*8), key+offset);
1624 	offset += (64 + key_t_value*8);
1625 	nettle_mpz_set_str_256_u(pubkey.g, (64 + key_t_value*8), key+offset);
1626 	offset += (64 + key_t_value*8);
1627 	nettle_mpz_set_str_256_u(pubkey.y, (64 + key_t_value*8), key+offset);
1628 
1629 	/* Digest content of "buf" and verify its DSA signature in "sigblock"*/
1630 	res = _digest_nettle(SHA1_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1631 						(unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1632 	res &= dsa_sha1_verify_digest(&pubkey, digest, &signature);
1633 
1634 	/* Clear and return */
1635 	nettle_dsa_signature_clear(&signature);
1636 	nettle_dsa_public_key_clear(&pubkey);
1637 	if (!res)
1638 		return "DSA signature verification failed";
1639 	else
1640 		return NULL;
1641 }
1642 #endif /* USE_DSA */
1643 
1644 static char *
1645 _verify_nettle_rsa(sldns_buffer* buf, unsigned int digest_size, char* sigblock,
1646 	unsigned int sigblock_len, uint8_t* key, unsigned int keylen)
1647 {
1648 	uint16_t exp_len = 0;
1649 	size_t exp_offset = 0, mod_offset = 0;
1650 	struct rsa_public_key pubkey;
1651 	mpz_t signature;
1652 	int res = 0;
1653 
1654 	/* RSA pubkey parsing as per RFC 3110 sec. 2 */
1655 	if( keylen <= 1) {
1656 		return "null RSA key";
1657 	}
1658 	if (key[0] != 0) {
1659 		/* 1-byte length */
1660 		exp_len = key[0];
1661 		exp_offset = 1;
1662 	} else {
1663 		/* 1-byte NUL + 2-bytes exponent length */
1664 		if (keylen < 3) {
1665 			return "incorrect RSA key length";
1666 		}
1667 		exp_len = READ_UINT16(key+1);
1668 		if (exp_len == 0)
1669 			return "null RSA exponent length";
1670 		exp_offset = 3;
1671 	}
1672 	/* Check that we are not over-running input length */
1673 	if (keylen < exp_offset + exp_len + 1) {
1674 		return "RSA key content shorter than expected";
1675 	}
1676 	mod_offset = exp_offset + exp_len;
1677 	nettle_rsa_public_key_init(&pubkey);
1678 	pubkey.size = keylen - mod_offset;
1679 	nettle_mpz_set_str_256_u(pubkey.e, exp_len, &key[exp_offset]);
1680 	nettle_mpz_set_str_256_u(pubkey.n, pubkey.size, &key[mod_offset]);
1681 
1682 	/* Digest content of "buf" and verify its RSA signature in "sigblock"*/
1683 	nettle_mpz_init_set_str_256_u(signature, sigblock_len, (uint8_t*)sigblock);
1684 	switch (digest_size) {
1685 		case SHA1_DIGEST_SIZE:
1686 		{
1687 			uint8_t digest[SHA1_DIGEST_SIZE];
1688 			res = _digest_nettle(SHA1_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1689 						(unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1690 			res &= rsa_sha1_verify_digest(&pubkey, digest, signature);
1691 			break;
1692 		}
1693 		case SHA256_DIGEST_SIZE:
1694 		{
1695 			uint8_t digest[SHA256_DIGEST_SIZE];
1696 			res = _digest_nettle(SHA256_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1697 						(unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1698 			res &= rsa_sha256_verify_digest(&pubkey, digest, signature);
1699 			break;
1700 		}
1701 		case SHA512_DIGEST_SIZE:
1702 		{
1703 			uint8_t digest[SHA512_DIGEST_SIZE];
1704 			res = _digest_nettle(SHA512_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1705 						(unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1706 			res &= rsa_sha512_verify_digest(&pubkey, digest, signature);
1707 			break;
1708 		}
1709 		default:
1710 			break;
1711 	}
1712 
1713 	/* Clear and return */
1714 	nettle_rsa_public_key_clear(&pubkey);
1715 	mpz_clear(signature);
1716 	if (!res) {
1717 		return "RSA signature verification failed";
1718 	} else {
1719 		return NULL;
1720 	}
1721 }
1722 
1723 #ifdef USE_ECDSA
1724 static char *
1725 _verify_nettle_ecdsa(sldns_buffer* buf, unsigned int digest_size, unsigned char* sigblock,
1726 	unsigned int sigblock_len, unsigned char* key, unsigned int keylen)
1727 {
1728 	int res = 0;
1729 	struct ecc_point pubkey;
1730 	struct dsa_signature signature;
1731 
1732 	/* Always matched strength, as per RFC 6605 sec. 1 */
1733 	if (sigblock_len != 2*digest_size || keylen != 2*digest_size) {
1734 		return "wrong ECDSA signature length";
1735 	}
1736 
1737 	/* Parse ECDSA signature as per RFC 6605 sec. 4 */
1738 	nettle_dsa_signature_init(&signature);
1739 	switch (digest_size) {
1740 		case SHA256_DIGEST_SIZE:
1741 		{
1742 			uint8_t digest[SHA256_DIGEST_SIZE];
1743 			mpz_t x, y;
1744 			nettle_ecc_point_init(&pubkey, nettle_get_secp_256r1());
1745 			nettle_mpz_init_set_str_256_u(x, SHA256_DIGEST_SIZE, key);
1746 			nettle_mpz_init_set_str_256_u(y, SHA256_DIGEST_SIZE, key+SHA256_DIGEST_SIZE);
1747 			nettle_mpz_set_str_256_u(signature.r, SHA256_DIGEST_SIZE, sigblock);
1748 			nettle_mpz_set_str_256_u(signature.s, SHA256_DIGEST_SIZE, sigblock+SHA256_DIGEST_SIZE);
1749 			res = _digest_nettle(SHA256_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1750 						(unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1751 			res &= nettle_ecc_point_set(&pubkey, x, y);
1752 			res &= nettle_ecdsa_verify (&pubkey, SHA256_DIGEST_SIZE, digest, &signature);
1753 			mpz_clear(x);
1754 			mpz_clear(y);
1755 			nettle_ecc_point_clear(&pubkey);
1756 			break;
1757 		}
1758 		case SHA384_DIGEST_SIZE:
1759 		{
1760 			uint8_t digest[SHA384_DIGEST_SIZE];
1761 			mpz_t x, y;
1762 			nettle_ecc_point_init(&pubkey, nettle_get_secp_384r1());
1763 			nettle_mpz_init_set_str_256_u(x, SHA384_DIGEST_SIZE, key);
1764 			nettle_mpz_init_set_str_256_u(y, SHA384_DIGEST_SIZE, key+SHA384_DIGEST_SIZE);
1765 			nettle_mpz_set_str_256_u(signature.r, SHA384_DIGEST_SIZE, sigblock);
1766 			nettle_mpz_set_str_256_u(signature.s, SHA384_DIGEST_SIZE, sigblock+SHA384_DIGEST_SIZE);
1767 			res = _digest_nettle(SHA384_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1768 						(unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1769 			res &= nettle_ecc_point_set(&pubkey, x, y);
1770 			res &= nettle_ecdsa_verify (&pubkey, SHA384_DIGEST_SIZE, digest, &signature);
1771 			mpz_clear(x);
1772 			mpz_clear(y);
1773 			nettle_ecc_point_clear(&pubkey);
1774 			break;
1775 		}
1776 		default:
1777 			return "unknown ECDSA algorithm";
1778 	}
1779 
1780 	/* Clear and return */
1781 	nettle_dsa_signature_clear(&signature);
1782 	if (!res)
1783 		return "ECDSA signature verification failed";
1784 	else
1785 		return NULL;
1786 }
1787 #endif
1788 
1789 #ifdef USE_ED25519
1790 static char *
1791 _verify_nettle_ed25519(sldns_buffer* buf, unsigned char* sigblock,
1792 	unsigned int sigblock_len, unsigned char* key, unsigned int keylen)
1793 {
1794 	int res = 0;
1795 
1796 	if(sigblock_len != ED25519_SIGNATURE_SIZE) {
1797 		return "wrong ED25519 signature length";
1798 	}
1799 	if(keylen != ED25519_KEY_SIZE) {
1800 		return "wrong ED25519 key length";
1801 	}
1802 
1803 	res = ed25519_sha512_verify((uint8_t*)key, sldns_buffer_limit(buf),
1804 		sldns_buffer_begin(buf), (uint8_t*)sigblock);
1805 
1806 	if (!res)
1807 		return "ED25519 signature verification failed";
1808 	else
1809 		return NULL;
1810 }
1811 #endif
1812 
1813 /**
1814  * Check a canonical sig+rrset and signature against a dnskey
1815  * @param buf: buffer with data to verify, the first rrsig part and the
1816  *	canonicalized rrset.
1817  * @param algo: DNSKEY algorithm.
1818  * @param sigblock: signature rdata field from RRSIG
1819  * @param sigblock_len: length of sigblock data.
1820  * @param key: public key data from DNSKEY RR.
1821  * @param keylen: length of keydata.
1822  * @param reason: bogus reason in more detail.
1823  * @return secure if verification succeeded, bogus on crypto failure,
1824  *	unchecked on format errors and alloc failures.
1825  */
1826 enum sec_status
1827 verify_canonrrset(sldns_buffer* buf, int algo, unsigned char* sigblock,
1828 	unsigned int sigblock_len, unsigned char* key, unsigned int keylen,
1829 	char** reason)
1830 {
1831 	unsigned int digest_size = 0;
1832 
1833 	if (sigblock_len == 0 || keylen == 0) {
1834 		*reason = "null signature";
1835 		return sec_status_bogus;
1836 	}
1837 
1838 #ifndef USE_DSA
1839 	if((algo == LDNS_DSA || algo == LDNS_DSA_NSEC3) &&(fake_dsa||fake_sha1))
1840 		return sec_status_secure;
1841 #endif
1842 #ifndef USE_SHA1
1843 	if(fake_sha1 && (algo == LDNS_DSA || algo == LDNS_DSA_NSEC3 || algo == LDNS_RSASHA1 || algo == LDNS_RSASHA1_NSEC3))
1844 		return sec_status_secure;
1845 #endif
1846 
1847 	switch(algo) {
1848 #if defined(USE_DSA) && defined(USE_SHA1)
1849 	case LDNS_DSA:
1850 	case LDNS_DSA_NSEC3:
1851 		*reason = _verify_nettle_dsa(buf, sigblock, sigblock_len, key, keylen);
1852 		if (*reason != NULL)
1853 			return sec_status_bogus;
1854 		else
1855 			return sec_status_secure;
1856 #endif /* USE_DSA */
1857 
1858 #ifdef USE_SHA1
1859 	case LDNS_RSASHA1:
1860 	case LDNS_RSASHA1_NSEC3:
1861 		digest_size = (digest_size ? digest_size : SHA1_DIGEST_SIZE);
1862 #endif
1863 		/* double fallthrough annotation to please gcc parser */
1864 		/* fallthrough */
1865 #ifdef USE_SHA2
1866 		/* fallthrough */
1867 	case LDNS_RSASHA256:
1868 		digest_size = (digest_size ? digest_size : SHA256_DIGEST_SIZE);
1869 		/* fallthrough */
1870 	case LDNS_RSASHA512:
1871 		digest_size = (digest_size ? digest_size : SHA512_DIGEST_SIZE);
1872 
1873 #endif
1874 		*reason = _verify_nettle_rsa(buf, digest_size, (char*)sigblock,
1875 						sigblock_len, key, keylen);
1876 		if (*reason != NULL)
1877 			return sec_status_bogus;
1878 		else
1879 			return sec_status_secure;
1880 
1881 #ifdef USE_ECDSA
1882 	case LDNS_ECDSAP256SHA256:
1883 		digest_size = (digest_size ? digest_size : SHA256_DIGEST_SIZE);
1884 		/* fallthrough */
1885 	case LDNS_ECDSAP384SHA384:
1886 		digest_size = (digest_size ? digest_size : SHA384_DIGEST_SIZE);
1887 		*reason = _verify_nettle_ecdsa(buf, digest_size, sigblock,
1888 						sigblock_len, key, keylen);
1889 		if (*reason != NULL)
1890 			return sec_status_bogus;
1891 		else
1892 			return sec_status_secure;
1893 #endif
1894 #ifdef USE_ED25519
1895 	case LDNS_ED25519:
1896 		*reason = _verify_nettle_ed25519(buf, sigblock, sigblock_len,
1897 			key, keylen);
1898 		if (*reason != NULL)
1899 			return sec_status_bogus;
1900 		else
1901 			return sec_status_secure;
1902 #endif
1903 	case LDNS_RSAMD5:
1904 	case LDNS_ECC_GOST:
1905 	default:
1906 		*reason = "unable to verify signature, unknown algorithm";
1907 		return sec_status_bogus;
1908 	}
1909 }
1910 
1911 #endif /* HAVE_SSL or HAVE_NSS or HAVE_NETTLE */
1912