xref: /freebsd/contrib/unbound/services/authzone.c (revision ea31d1a5c490193348e0b71fdda6d08a2c80a2eb)
1 /*
2  * services/authzone.c - authoritative zone that is locally hosted.
3  *
4  * Copyright (c) 2017, NLnet Labs. All rights reserved.
5  *
6  * This software is open source.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright notice,
13  * this list of conditions and the following disclaimer.
14  *
15  * Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  *
19  * Neither the name of the NLNET LABS nor the names of its contributors may
20  * be used to endorse or promote products derived from this software without
21  * specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 /**
37  * \file
38  *
39  * This file contains the functions for an authority zone.  This zone
40  * is queried by the iterator, just like a stub or forward zone, but then
41  * the data is locally held.
42  */
43 
44 #include "config.h"
45 #include "services/authzone.h"
46 #include "util/data/dname.h"
47 #include "util/data/msgparse.h"
48 #include "util/data/msgreply.h"
49 #include "util/data/msgencode.h"
50 #include "util/data/packed_rrset.h"
51 #include "util/regional.h"
52 #include "util/net_help.h"
53 #include "util/netevent.h"
54 #include "util/config_file.h"
55 #include "util/log.h"
56 #include "util/module.h"
57 #include "util/random.h"
58 #include "services/cache/dns.h"
59 #include "services/outside_network.h"
60 #include "services/listen_dnsport.h"
61 #include "services/mesh.h"
62 #include "sldns/rrdef.h"
63 #include "sldns/pkthdr.h"
64 #include "sldns/sbuffer.h"
65 #include "sldns/str2wire.h"
66 #include "sldns/wire2str.h"
67 #include "sldns/parseutil.h"
68 #include "sldns/keyraw.h"
69 #include "validator/val_nsec3.h"
70 #include "validator/val_nsec.h"
71 #include "validator/val_secalgo.h"
72 #include "validator/val_sigcrypt.h"
73 #include "validator/val_anchor.h"
74 #include "validator/val_utils.h"
75 #include <ctype.h>
76 
77 /** bytes to use for NSEC3 hash buffer. 20 for sha1 */
78 #define N3HASHBUFLEN 32
79 /** max number of CNAMEs we are willing to follow (in one answer) */
80 #define MAX_CNAME_CHAIN 8
81 /** timeout for probe packets for SOA */
82 #define AUTH_PROBE_TIMEOUT 100 /* msec */
83 /** when to stop with SOA probes (when exponential timeouts exceed this) */
84 #define AUTH_PROBE_TIMEOUT_STOP 1000 /* msec */
85 /* auth transfer timeout for TCP connections, in msec */
86 #define AUTH_TRANSFER_TIMEOUT 10000 /* msec */
87 /* auth transfer max backoff for failed tranfers and probes */
88 #define AUTH_TRANSFER_MAX_BACKOFF 86400 /* sec */
89 /* auth http port number */
90 #define AUTH_HTTP_PORT 80
91 /* auth https port number */
92 #define AUTH_HTTPS_PORT 443
93 /* max depth for nested $INCLUDEs */
94 #define MAX_INCLUDE_DEPTH 10
95 /** number of timeouts before we fallback from IXFR to AXFR,
96  * because some versions of servers (eg. dnsmasq) drop IXFR packets. */
97 #define NUM_TIMEOUTS_FALLBACK_IXFR 3
98 
99 /** pick up nextprobe task to start waiting to perform transfer actions */
100 static void xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
101 	int failure, int lookup_only);
102 /** move to sending the probe packets, next if fails. task_probe */
103 static void xfr_probe_send_or_end(struct auth_xfer* xfr,
104 	struct module_env* env);
105 /** pick up probe task with specified(or NULL) destination first,
106  * or transfer task if nothing to probe, or false if already in progress */
107 static int xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
108 	struct auth_master* spec);
109 /** delete xfer structure (not its tree entry) */
110 static void auth_xfer_delete(struct auth_xfer* xfr);
111 
112 /** create new dns_msg */
113 static struct dns_msg*
114 msg_create(struct regional* region, struct query_info* qinfo)
115 {
116 	struct dns_msg* msg = (struct dns_msg*)regional_alloc(region,
117 		sizeof(struct dns_msg));
118 	if(!msg)
119 		return NULL;
120 	msg->qinfo.qname = regional_alloc_init(region, qinfo->qname,
121 		qinfo->qname_len);
122 	if(!msg->qinfo.qname)
123 		return NULL;
124 	msg->qinfo.qname_len = qinfo->qname_len;
125 	msg->qinfo.qtype = qinfo->qtype;
126 	msg->qinfo.qclass = qinfo->qclass;
127 	msg->qinfo.local_alias = NULL;
128 	/* non-packed reply_info, because it needs to grow the array */
129 	msg->rep = (struct reply_info*)regional_alloc_zero(region,
130 		sizeof(struct reply_info)-sizeof(struct rrset_ref));
131 	if(!msg->rep)
132 		return NULL;
133 	msg->rep->flags = (uint16_t)(BIT_QR | BIT_AA);
134 	msg->rep->authoritative = 1;
135 	msg->rep->qdcount = 1;
136 	/* rrsets is NULL, no rrsets yet */
137 	return msg;
138 }
139 
140 /** grow rrset array by one in msg */
141 static int
142 msg_grow_array(struct regional* region, struct dns_msg* msg)
143 {
144 	if(msg->rep->rrsets == NULL) {
145 		msg->rep->rrsets = regional_alloc_zero(region,
146 			sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
147 		if(!msg->rep->rrsets)
148 			return 0;
149 	} else {
150 		struct ub_packed_rrset_key** rrsets_old = msg->rep->rrsets;
151 		msg->rep->rrsets = regional_alloc_zero(region,
152 			sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
153 		if(!msg->rep->rrsets)
154 			return 0;
155 		memmove(msg->rep->rrsets, rrsets_old,
156 			sizeof(struct ub_packed_rrset_key*)*msg->rep->rrset_count);
157 	}
158 	return 1;
159 }
160 
161 /** get ttl of rrset */
162 static time_t
163 get_rrset_ttl(struct ub_packed_rrset_key* k)
164 {
165 	struct packed_rrset_data* d = (struct packed_rrset_data*)
166 		k->entry.data;
167 	return d->ttl;
168 }
169 
170 /** Copy rrset into region from domain-datanode and packet rrset */
171 static struct ub_packed_rrset_key*
172 auth_packed_rrset_copy_region(struct auth_zone* z, struct auth_data* node,
173 	struct auth_rrset* rrset, struct regional* region, time_t adjust)
174 {
175 	struct ub_packed_rrset_key key;
176 	memset(&key, 0, sizeof(key));
177 	key.entry.key = &key;
178 	key.entry.data = rrset->data;
179 	key.rk.dname = node->name;
180 	key.rk.dname_len = node->namelen;
181 	key.rk.type = htons(rrset->type);
182 	key.rk.rrset_class = htons(z->dclass);
183 	key.entry.hash = rrset_key_hash(&key.rk);
184 	return packed_rrset_copy_region(&key, region, adjust);
185 }
186 
187 /** fix up msg->rep TTL and prefetch ttl */
188 static void
189 msg_ttl(struct dns_msg* msg)
190 {
191 	if(msg->rep->rrset_count == 0) return;
192 	if(msg->rep->rrset_count == 1) {
193 		msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
194 		msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
195 		msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
196 	} else if(get_rrset_ttl(msg->rep->rrsets[msg->rep->rrset_count-1]) <
197 		msg->rep->ttl) {
198 		msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[
199 			msg->rep->rrset_count-1]);
200 		msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
201 		msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
202 	}
203 }
204 
205 /** see if rrset is a duplicate in the answer message */
206 static int
207 msg_rrset_duplicate(struct dns_msg* msg, uint8_t* nm, size_t nmlen,
208 	uint16_t type, uint16_t dclass)
209 {
210 	size_t i;
211 	for(i=0; i<msg->rep->rrset_count; i++) {
212 		struct ub_packed_rrset_key* k = msg->rep->rrsets[i];
213 		if(ntohs(k->rk.type) == type && k->rk.dname_len == nmlen &&
214 			ntohs(k->rk.rrset_class) == dclass &&
215 			query_dname_compare(k->rk.dname, nm) == 0)
216 			return 1;
217 	}
218 	return 0;
219 }
220 
221 /** add rrset to answer section (no auth, add rrsets yet) */
222 static int
223 msg_add_rrset_an(struct auth_zone* z, struct regional* region,
224 	struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
225 {
226 	log_assert(msg->rep->ns_numrrsets == 0);
227 	log_assert(msg->rep->ar_numrrsets == 0);
228 	if(!rrset || !node)
229 		return 1;
230 	if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
231 		z->dclass))
232 		return 1;
233 	/* grow array */
234 	if(!msg_grow_array(region, msg))
235 		return 0;
236 	/* copy it */
237 	if(!(msg->rep->rrsets[msg->rep->rrset_count] =
238 		auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
239 		return 0;
240 	msg->rep->rrset_count++;
241 	msg->rep->an_numrrsets++;
242 	msg_ttl(msg);
243 	return 1;
244 }
245 
246 /** add rrset to authority section (no additonal section rrsets yet) */
247 static int
248 msg_add_rrset_ns(struct auth_zone* z, struct regional* region,
249 	struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
250 {
251 	log_assert(msg->rep->ar_numrrsets == 0);
252 	if(!rrset || !node)
253 		return 1;
254 	if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
255 		z->dclass))
256 		return 1;
257 	/* grow array */
258 	if(!msg_grow_array(region, msg))
259 		return 0;
260 	/* copy it */
261 	if(!(msg->rep->rrsets[msg->rep->rrset_count] =
262 		auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
263 		return 0;
264 	msg->rep->rrset_count++;
265 	msg->rep->ns_numrrsets++;
266 	msg_ttl(msg);
267 	return 1;
268 }
269 
270 /** add rrset to additional section */
271 static int
272 msg_add_rrset_ar(struct auth_zone* z, struct regional* region,
273 	struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
274 {
275 	if(!rrset || !node)
276 		return 1;
277 	if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
278 		z->dclass))
279 		return 1;
280 	/* grow array */
281 	if(!msg_grow_array(region, msg))
282 		return 0;
283 	/* copy it */
284 	if(!(msg->rep->rrsets[msg->rep->rrset_count] =
285 		auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
286 		return 0;
287 	msg->rep->rrset_count++;
288 	msg->rep->ar_numrrsets++;
289 	msg_ttl(msg);
290 	return 1;
291 }
292 
293 struct auth_zones* auth_zones_create(void)
294 {
295 	struct auth_zones* az = (struct auth_zones*)calloc(1, sizeof(*az));
296 	if(!az) {
297 		log_err("out of memory");
298 		return NULL;
299 	}
300 	rbtree_init(&az->ztree, &auth_zone_cmp);
301 	rbtree_init(&az->xtree, &auth_xfer_cmp);
302 	lock_rw_init(&az->lock);
303 	lock_protect(&az->lock, &az->ztree, sizeof(az->ztree));
304 	lock_protect(&az->lock, &az->xtree, sizeof(az->xtree));
305 	/* also lock protects the rbnode's in struct auth_zone, auth_xfer */
306 	lock_rw_init(&az->rpz_lock);
307 	lock_protect(&az->rpz_lock, &az->rpz_first, sizeof(az->rpz_first));
308 	return az;
309 }
310 
311 int auth_zone_cmp(const void* z1, const void* z2)
312 {
313 	/* first sort on class, so that hierarchy can be maintained within
314 	 * a class */
315 	struct auth_zone* a = (struct auth_zone*)z1;
316 	struct auth_zone* b = (struct auth_zone*)z2;
317 	int m;
318 	if(a->dclass != b->dclass) {
319 		if(a->dclass < b->dclass)
320 			return -1;
321 		return 1;
322 	}
323 	/* sorted such that higher zones sort before lower zones (their
324 	 * contents) */
325 	return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
326 }
327 
328 int auth_data_cmp(const void* z1, const void* z2)
329 {
330 	struct auth_data* a = (struct auth_data*)z1;
331 	struct auth_data* b = (struct auth_data*)z2;
332 	int m;
333 	/* canonical sort, because DNSSEC needs that */
334 	return dname_canon_lab_cmp(a->name, a->namelabs, b->name,
335 		b->namelabs, &m);
336 }
337 
338 int auth_xfer_cmp(const void* z1, const void* z2)
339 {
340 	/* first sort on class, so that hierarchy can be maintained within
341 	 * a class */
342 	struct auth_xfer* a = (struct auth_xfer*)z1;
343 	struct auth_xfer* b = (struct auth_xfer*)z2;
344 	int m;
345 	if(a->dclass != b->dclass) {
346 		if(a->dclass < b->dclass)
347 			return -1;
348 		return 1;
349 	}
350 	/* sorted such that higher zones sort before lower zones (their
351 	 * contents) */
352 	return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
353 }
354 
355 /** delete auth rrset node */
356 static void
357 auth_rrset_delete(struct auth_rrset* rrset)
358 {
359 	if(!rrset) return;
360 	free(rrset->data);
361 	free(rrset);
362 }
363 
364 /** delete auth data domain node */
365 static void
366 auth_data_delete(struct auth_data* n)
367 {
368 	struct auth_rrset* p, *np;
369 	if(!n) return;
370 	p = n->rrsets;
371 	while(p) {
372 		np = p->next;
373 		auth_rrset_delete(p);
374 		p = np;
375 	}
376 	free(n->name);
377 	free(n);
378 }
379 
380 /** helper traverse to delete zones */
381 static void
382 auth_data_del(rbnode_type* n, void* ATTR_UNUSED(arg))
383 {
384 	struct auth_data* z = (struct auth_data*)n->key;
385 	auth_data_delete(z);
386 }
387 
388 /** delete an auth zone structure (tree remove must be done elsewhere) */
389 static void
390 auth_zone_delete(struct auth_zone* z, struct auth_zones* az)
391 {
392 	if(!z) return;
393 	lock_rw_destroy(&z->lock);
394 	traverse_postorder(&z->data, auth_data_del, NULL);
395 
396 	if(az && z->rpz) {
397 		/* keep RPZ linked list intact */
398 		lock_rw_wrlock(&az->rpz_lock);
399 		if(z->rpz_az_prev)
400 			z->rpz_az_prev->rpz_az_next = z->rpz_az_next;
401 		else
402 			az->rpz_first = z->rpz_az_next;
403 		if(z->rpz_az_next)
404 			z->rpz_az_next->rpz_az_prev = z->rpz_az_prev;
405 		lock_rw_unlock(&az->rpz_lock);
406 	}
407 	if(z->rpz)
408 		rpz_delete(z->rpz);
409 	free(z->name);
410 	free(z->zonefile);
411 	free(z);
412 }
413 
414 struct auth_zone*
415 auth_zone_create(struct auth_zones* az, uint8_t* nm, size_t nmlen,
416 	uint16_t dclass)
417 {
418 	struct auth_zone* z = (struct auth_zone*)calloc(1, sizeof(*z));
419 	if(!z) {
420 		return NULL;
421 	}
422 	z->node.key = z;
423 	z->dclass = dclass;
424 	z->namelen = nmlen;
425 	z->namelabs = dname_count_labels(nm);
426 	z->name = memdup(nm, nmlen);
427 	if(!z->name) {
428 		free(z);
429 		return NULL;
430 	}
431 	rbtree_init(&z->data, &auth_data_cmp);
432 	lock_rw_init(&z->lock);
433 	lock_protect(&z->lock, &z->name, sizeof(*z)-sizeof(rbnode_type)-
434 			sizeof(&z->rpz_az_next)-sizeof(&z->rpz_az_prev));
435 	lock_rw_wrlock(&z->lock);
436 	/* z lock protects all, except rbtree itself and the rpz linked list
437 	 * pointers, which are protected using az->lock */
438 	if(!rbtree_insert(&az->ztree, &z->node)) {
439 		lock_rw_unlock(&z->lock);
440 		auth_zone_delete(z, NULL);
441 		log_warn("duplicate auth zone");
442 		return NULL;
443 	}
444 	return z;
445 }
446 
447 struct auth_zone*
448 auth_zone_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
449 	uint16_t dclass)
450 {
451 	struct auth_zone key;
452 	key.node.key = &key;
453 	key.dclass = dclass;
454 	key.name = nm;
455 	key.namelen = nmlen;
456 	key.namelabs = dname_count_labels(nm);
457 	return (struct auth_zone*)rbtree_search(&az->ztree, &key);
458 }
459 
460 struct auth_xfer*
461 auth_xfer_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
462 	uint16_t dclass)
463 {
464 	struct auth_xfer key;
465 	key.node.key = &key;
466 	key.dclass = dclass;
467 	key.name = nm;
468 	key.namelen = nmlen;
469 	key.namelabs = dname_count_labels(nm);
470 	return (struct auth_xfer*)rbtree_search(&az->xtree, &key);
471 }
472 
473 /** find an auth zone or sorted less-or-equal, return true if exact */
474 static int
475 auth_zone_find_less_equal(struct auth_zones* az, uint8_t* nm, size_t nmlen,
476 	uint16_t dclass, struct auth_zone** z)
477 {
478 	struct auth_zone key;
479 	key.node.key = &key;
480 	key.dclass = dclass;
481 	key.name = nm;
482 	key.namelen = nmlen;
483 	key.namelabs = dname_count_labels(nm);
484 	return rbtree_find_less_equal(&az->ztree, &key, (rbnode_type**)z);
485 }
486 
487 
488 /** find the auth zone that is above the given name */
489 struct auth_zone*
490 auth_zones_find_zone(struct auth_zones* az, uint8_t* name, size_t name_len,
491 	uint16_t dclass)
492 {
493 	uint8_t* nm = name;
494 	size_t nmlen = name_len;
495 	struct auth_zone* z;
496 	if(auth_zone_find_less_equal(az, nm, nmlen, dclass, &z)) {
497 		/* exact match */
498 		return z;
499 	} else {
500 		/* less-or-nothing */
501 		if(!z) return NULL; /* nothing smaller, nothing above it */
502 		/* we found smaller name; smaller may be above the name,
503 		 * but not below it. */
504 		nm = dname_get_shared_topdomain(z->name, name);
505 		dname_count_size_labels(nm, &nmlen);
506 		z = NULL;
507 	}
508 
509 	/* search up */
510 	while(!z) {
511 		z = auth_zone_find(az, nm, nmlen, dclass);
512 		if(z) return z;
513 		if(dname_is_root(nm)) break;
514 		dname_remove_label(&nm, &nmlen);
515 	}
516 	return NULL;
517 }
518 
519 /** find or create zone with name str. caller must have lock on az.
520  * returns a wrlocked zone */
521 static struct auth_zone*
522 auth_zones_find_or_add_zone(struct auth_zones* az, char* name)
523 {
524 	uint8_t nm[LDNS_MAX_DOMAINLEN+1];
525 	size_t nmlen = sizeof(nm);
526 	struct auth_zone* z;
527 
528 	if(sldns_str2wire_dname_buf(name, nm, &nmlen) != 0) {
529 		log_err("cannot parse auth zone name: %s", name);
530 		return 0;
531 	}
532 	z = auth_zone_find(az, nm, nmlen, LDNS_RR_CLASS_IN);
533 	if(!z) {
534 		/* not found, create the zone */
535 		z = auth_zone_create(az, nm, nmlen, LDNS_RR_CLASS_IN);
536 	} else {
537 		lock_rw_wrlock(&z->lock);
538 	}
539 	return z;
540 }
541 
542 /** find or create xfer zone with name str. caller must have lock on az.
543  * returns a locked xfer */
544 static struct auth_xfer*
545 auth_zones_find_or_add_xfer(struct auth_zones* az, struct auth_zone* z)
546 {
547 	struct auth_xfer* x;
548 	x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
549 	if(!x) {
550 		/* not found, create the zone */
551 		x = auth_xfer_create(az, z);
552 	} else {
553 		lock_basic_lock(&x->lock);
554 	}
555 	return x;
556 }
557 
558 int
559 auth_zone_set_zonefile(struct auth_zone* z, char* zonefile)
560 {
561 	if(z->zonefile) free(z->zonefile);
562 	if(zonefile == NULL) {
563 		z->zonefile = NULL;
564 	} else {
565 		z->zonefile = strdup(zonefile);
566 		if(!z->zonefile) {
567 			log_err("malloc failure");
568 			return 0;
569 		}
570 	}
571 	return 1;
572 }
573 
574 /** set auth zone fallback. caller must have lock on zone */
575 int
576 auth_zone_set_fallback(struct auth_zone* z, char* fallbackstr)
577 {
578 	if(strcmp(fallbackstr, "yes") != 0 && strcmp(fallbackstr, "no") != 0){
579 		log_err("auth zone fallback, expected yes or no, got %s",
580 			fallbackstr);
581 		return 0;
582 	}
583 	z->fallback_enabled = (strcmp(fallbackstr, "yes")==0);
584 	return 1;
585 }
586 
587 /** create domain with the given name */
588 static struct auth_data*
589 az_domain_create(struct auth_zone* z, uint8_t* nm, size_t nmlen)
590 {
591 	struct auth_data* n = (struct auth_data*)malloc(sizeof(*n));
592 	if(!n) return NULL;
593 	memset(n, 0, sizeof(*n));
594 	n->node.key = n;
595 	n->name = memdup(nm, nmlen);
596 	if(!n->name) {
597 		free(n);
598 		return NULL;
599 	}
600 	n->namelen = nmlen;
601 	n->namelabs = dname_count_labels(nm);
602 	if(!rbtree_insert(&z->data, &n->node)) {
603 		log_warn("duplicate auth domain name");
604 		free(n->name);
605 		free(n);
606 		return NULL;
607 	}
608 	return n;
609 }
610 
611 /** find domain with exactly the given name */
612 static struct auth_data*
613 az_find_name(struct auth_zone* z, uint8_t* nm, size_t nmlen)
614 {
615 	struct auth_zone key;
616 	key.node.key = &key;
617 	key.name = nm;
618 	key.namelen = nmlen;
619 	key.namelabs = dname_count_labels(nm);
620 	return (struct auth_data*)rbtree_search(&z->data, &key);
621 }
622 
623 /** Find domain name (or closest match) */
624 static void
625 az_find_domain(struct auth_zone* z, struct query_info* qinfo, int* node_exact,
626 	struct auth_data** node)
627 {
628 	struct auth_zone key;
629 	key.node.key = &key;
630 	key.name = qinfo->qname;
631 	key.namelen = qinfo->qname_len;
632 	key.namelabs = dname_count_labels(key.name);
633 	*node_exact = rbtree_find_less_equal(&z->data, &key,
634 		(rbnode_type**)node);
635 }
636 
637 /** find or create domain with name in zone */
638 static struct auth_data*
639 az_domain_find_or_create(struct auth_zone* z, uint8_t* dname,
640 	size_t dname_len)
641 {
642 	struct auth_data* n = az_find_name(z, dname, dname_len);
643 	if(!n) {
644 		n = az_domain_create(z, dname, dname_len);
645 	}
646 	return n;
647 }
648 
649 /** find rrset of given type in the domain */
650 static struct auth_rrset*
651 az_domain_rrset(struct auth_data* n, uint16_t t)
652 {
653 	struct auth_rrset* rrset;
654 	if(!n) return NULL;
655 	rrset = n->rrsets;
656 	while(rrset) {
657 		if(rrset->type == t)
658 			return rrset;
659 		rrset = rrset->next;
660 	}
661 	return NULL;
662 }
663 
664 /** remove rrset of this type from domain */
665 static void
666 domain_remove_rrset(struct auth_data* node, uint16_t rr_type)
667 {
668 	struct auth_rrset* rrset, *prev;
669 	if(!node) return;
670 	prev = NULL;
671 	rrset = node->rrsets;
672 	while(rrset) {
673 		if(rrset->type == rr_type) {
674 			/* found it, now delete it */
675 			if(prev) prev->next = rrset->next;
676 			else	node->rrsets = rrset->next;
677 			auth_rrset_delete(rrset);
678 			return;
679 		}
680 		prev = rrset;
681 		rrset = rrset->next;
682 	}
683 }
684 
685 /** find an rrsig index in the rrset.  returns true if found */
686 static int
687 az_rrset_find_rrsig(struct packed_rrset_data* d, uint8_t* rdata, size_t len,
688 	size_t* index)
689 {
690 	size_t i;
691 	for(i=d->count; i<d->count + d->rrsig_count; i++) {
692 		if(d->rr_len[i] != len)
693 			continue;
694 		if(memcmp(d->rr_data[i], rdata, len) == 0) {
695 			*index = i;
696 			return 1;
697 		}
698 	}
699 	return 0;
700 }
701 
702 /** see if rdata is duplicate */
703 static int
704 rdata_duplicate(struct packed_rrset_data* d, uint8_t* rdata, size_t len)
705 {
706 	size_t i;
707 	for(i=0; i<d->count + d->rrsig_count; i++) {
708 		if(d->rr_len[i] != len)
709 			continue;
710 		if(memcmp(d->rr_data[i], rdata, len) == 0)
711 			return 1;
712 	}
713 	return 0;
714 }
715 
716 /** get rrsig type covered from rdata.
717  * @param rdata: rdata in wireformat, starting with 16bit rdlength.
718  * @param rdatalen: length of rdata buffer.
719  * @return type covered (or 0).
720  */
721 static uint16_t
722 rrsig_rdata_get_type_covered(uint8_t* rdata, size_t rdatalen)
723 {
724 	if(rdatalen < 4)
725 		return 0;
726 	return sldns_read_uint16(rdata+2);
727 }
728 
729 /** remove RR from existing RRset. Also sig, if it is a signature.
730  * reallocates the packed rrset for a new one, false on alloc failure */
731 static int
732 rrset_remove_rr(struct auth_rrset* rrset, size_t index)
733 {
734 	struct packed_rrset_data* d, *old = rrset->data;
735 	size_t i;
736 	if(index >= old->count + old->rrsig_count)
737 		return 0; /* index out of bounds */
738 	d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old) - (
739 		sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t) +
740 		old->rr_len[index]));
741 	if(!d) {
742 		log_err("malloc failure");
743 		return 0;
744 	}
745 	d->ttl = old->ttl;
746 	d->count = old->count;
747 	d->rrsig_count = old->rrsig_count;
748 	if(index < d->count) d->count--;
749 	else d->rrsig_count--;
750 	d->trust = old->trust;
751 	d->security = old->security;
752 
753 	/* set rr_len, needed for ptr_fixup */
754 	d->rr_len = (size_t*)((uint8_t*)d +
755 		sizeof(struct packed_rrset_data));
756 	if(index > 0)
757 		memmove(d->rr_len, old->rr_len, (index)*sizeof(size_t));
758 	if(index+1 < old->count+old->rrsig_count)
759 		memmove(&d->rr_len[index], &old->rr_len[index+1],
760 		(old->count+old->rrsig_count - (index+1))*sizeof(size_t));
761 	packed_rrset_ptr_fixup(d);
762 
763 	/* move over ttls */
764 	if(index > 0)
765 		memmove(d->rr_ttl, old->rr_ttl, (index)*sizeof(time_t));
766 	if(index+1 < old->count+old->rrsig_count)
767 		memmove(&d->rr_ttl[index], &old->rr_ttl[index+1],
768 		(old->count+old->rrsig_count - (index+1))*sizeof(time_t));
769 
770 	/* move over rr_data */
771 	for(i=0; i<d->count+d->rrsig_count; i++) {
772 		size_t oldi;
773 		if(i < index) oldi = i;
774 		else oldi = i+1;
775 		memmove(d->rr_data[i], old->rr_data[oldi], d->rr_len[i]);
776 	}
777 
778 	/* recalc ttl (lowest of remaining RR ttls) */
779 	if(d->count + d->rrsig_count > 0)
780 		d->ttl = d->rr_ttl[0];
781 	for(i=0; i<d->count+d->rrsig_count; i++) {
782 		if(d->rr_ttl[i] < d->ttl)
783 			d->ttl = d->rr_ttl[i];
784 	}
785 
786 	free(rrset->data);
787 	rrset->data = d;
788 	return 1;
789 }
790 
791 /** add RR to existing RRset. If insert_sig is true, add to rrsigs.
792  * This reallocates the packed rrset for a new one */
793 static int
794 rrset_add_rr(struct auth_rrset* rrset, uint32_t rr_ttl, uint8_t* rdata,
795 	size_t rdatalen, int insert_sig)
796 {
797 	struct packed_rrset_data* d, *old = rrset->data;
798 	size_t total, old_total;
799 
800 	d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
801 		+ sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t)
802 		+ rdatalen);
803 	if(!d) {
804 		log_err("out of memory");
805 		return 0;
806 	}
807 	/* copy base values */
808 	memcpy(d, old, sizeof(struct packed_rrset_data));
809 	if(!insert_sig) {
810 		d->count++;
811 	} else {
812 		d->rrsig_count++;
813 	}
814 	old_total = old->count + old->rrsig_count;
815 	total = d->count + d->rrsig_count;
816 	/* set rr_len, needed for ptr_fixup */
817 	d->rr_len = (size_t*)((uint8_t*)d +
818 		sizeof(struct packed_rrset_data));
819 	if(old->count != 0)
820 		memmove(d->rr_len, old->rr_len, old->count*sizeof(size_t));
821 	if(old->rrsig_count != 0)
822 		memmove(d->rr_len+d->count, old->rr_len+old->count,
823 			old->rrsig_count*sizeof(size_t));
824 	if(!insert_sig)
825 		d->rr_len[d->count-1] = rdatalen;
826 	else	d->rr_len[total-1] = rdatalen;
827 	packed_rrset_ptr_fixup(d);
828 	if((time_t)rr_ttl < d->ttl)
829 		d->ttl = rr_ttl;
830 
831 	/* copy old values into new array */
832 	if(old->count != 0) {
833 		memmove(d->rr_ttl, old->rr_ttl, old->count*sizeof(time_t));
834 		/* all the old rr pieces are allocated sequential, so we
835 		 * can copy them in one go */
836 		memmove(d->rr_data[0], old->rr_data[0],
837 			(old->rr_data[old->count-1] - old->rr_data[0]) +
838 			old->rr_len[old->count-1]);
839 	}
840 	if(old->rrsig_count != 0) {
841 		memmove(d->rr_ttl+d->count, old->rr_ttl+old->count,
842 			old->rrsig_count*sizeof(time_t));
843 		memmove(d->rr_data[d->count], old->rr_data[old->count],
844 			(old->rr_data[old_total-1] - old->rr_data[old->count]) +
845 			old->rr_len[old_total-1]);
846 	}
847 
848 	/* insert new value */
849 	if(!insert_sig) {
850 		d->rr_ttl[d->count-1] = rr_ttl;
851 		memmove(d->rr_data[d->count-1], rdata, rdatalen);
852 	} else {
853 		d->rr_ttl[total-1] = rr_ttl;
854 		memmove(d->rr_data[total-1], rdata, rdatalen);
855 	}
856 
857 	rrset->data = d;
858 	free(old);
859 	return 1;
860 }
861 
862 /** Create new rrset for node with packed rrset with one RR element */
863 static struct auth_rrset*
864 rrset_create(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
865 	uint8_t* rdata, size_t rdatalen)
866 {
867 	struct auth_rrset* rrset = (struct auth_rrset*)calloc(1,
868 		sizeof(*rrset));
869 	struct auth_rrset* p, *prev;
870 	struct packed_rrset_data* d;
871 	if(!rrset) {
872 		log_err("out of memory");
873 		return NULL;
874 	}
875 	rrset->type = rr_type;
876 
877 	/* the rrset data structure, with one RR */
878 	d = (struct packed_rrset_data*)calloc(1,
879 		sizeof(struct packed_rrset_data) + sizeof(size_t) +
880 		sizeof(uint8_t*) + sizeof(time_t) + rdatalen);
881 	if(!d) {
882 		free(rrset);
883 		log_err("out of memory");
884 		return NULL;
885 	}
886 	rrset->data = d;
887 	d->ttl = rr_ttl;
888 	d->trust = rrset_trust_prim_noglue;
889 	d->rr_len = (size_t*)((uint8_t*)d + sizeof(struct packed_rrset_data));
890 	d->rr_data = (uint8_t**)&(d->rr_len[1]);
891 	d->rr_ttl = (time_t*)&(d->rr_data[1]);
892 	d->rr_data[0] = (uint8_t*)&(d->rr_ttl[1]);
893 
894 	/* insert the RR */
895 	d->rr_len[0] = rdatalen;
896 	d->rr_ttl[0] = rr_ttl;
897 	memmove(d->rr_data[0], rdata, rdatalen);
898 	d->count++;
899 
900 	/* insert rrset into linked list for domain */
901 	/* find sorted place to link the rrset into the list */
902 	prev = NULL;
903 	p = node->rrsets;
904 	while(p && p->type<=rr_type) {
905 		prev = p;
906 		p = p->next;
907 	}
908 	/* so, prev is smaller, and p is larger than rr_type */
909 	rrset->next = p;
910 	if(prev) prev->next = rrset;
911 	else node->rrsets = rrset;
912 	return rrset;
913 }
914 
915 /** count number (and size) of rrsigs that cover a type */
916 static size_t
917 rrsig_num_that_cover(struct auth_rrset* rrsig, uint16_t rr_type, size_t* sigsz)
918 {
919 	struct packed_rrset_data* d = rrsig->data;
920 	size_t i, num = 0;
921 	*sigsz = 0;
922 	log_assert(d && rrsig->type == LDNS_RR_TYPE_RRSIG);
923 	for(i=0; i<d->count+d->rrsig_count; i++) {
924 		if(rrsig_rdata_get_type_covered(d->rr_data[i],
925 			d->rr_len[i]) == rr_type) {
926 			num++;
927 			(*sigsz) += d->rr_len[i];
928 		}
929 	}
930 	return num;
931 }
932 
933 /** See if rrsig set has covered sigs for rrset and move them over */
934 static int
935 rrset_moveover_rrsigs(struct auth_data* node, uint16_t rr_type,
936 	struct auth_rrset* rrset, struct auth_rrset* rrsig)
937 {
938 	size_t sigs, sigsz, i, j, total;
939 	struct packed_rrset_data* sigold = rrsig->data;
940 	struct packed_rrset_data* old = rrset->data;
941 	struct packed_rrset_data* d, *sigd;
942 
943 	log_assert(rrset->type == rr_type);
944 	log_assert(rrsig->type == LDNS_RR_TYPE_RRSIG);
945 	sigs = rrsig_num_that_cover(rrsig, rr_type, &sigsz);
946 	if(sigs == 0) {
947 		/* 0 rrsigs to move over, done */
948 		return 1;
949 	}
950 
951 	/* allocate rrset sigsz larger for extra sigs elements, and
952 	 * allocate rrsig sigsz smaller for less sigs elements. */
953 	d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
954 		+ sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
955 		+ sigsz);
956 	if(!d) {
957 		log_err("out of memory");
958 		return 0;
959 	}
960 	/* copy base values */
961 	total = old->count + old->rrsig_count;
962 	memcpy(d, old, sizeof(struct packed_rrset_data));
963 	d->rrsig_count += sigs;
964 	/* setup rr_len */
965 	d->rr_len = (size_t*)((uint8_t*)d +
966 		sizeof(struct packed_rrset_data));
967 	if(total != 0)
968 		memmove(d->rr_len, old->rr_len, total*sizeof(size_t));
969 	j = d->count+d->rrsig_count-sigs;
970 	for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
971 		if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
972 			sigold->rr_len[i]) == rr_type) {
973 			d->rr_len[j] = sigold->rr_len[i];
974 			j++;
975 		}
976 	}
977 	packed_rrset_ptr_fixup(d);
978 
979 	/* copy old values into new array */
980 	if(total != 0) {
981 		memmove(d->rr_ttl, old->rr_ttl, total*sizeof(time_t));
982 		/* all the old rr pieces are allocated sequential, so we
983 		 * can copy them in one go */
984 		memmove(d->rr_data[0], old->rr_data[0],
985 			(old->rr_data[total-1] - old->rr_data[0]) +
986 			old->rr_len[total-1]);
987 	}
988 
989 	/* move over the rrsigs to the larger rrset*/
990 	j = d->count+d->rrsig_count-sigs;
991 	for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
992 		if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
993 			sigold->rr_len[i]) == rr_type) {
994 			/* move this one over to location j */
995 			d->rr_ttl[j] = sigold->rr_ttl[i];
996 			memmove(d->rr_data[j], sigold->rr_data[i],
997 				sigold->rr_len[i]);
998 			if(d->rr_ttl[j] < d->ttl)
999 				d->ttl = d->rr_ttl[j];
1000 			j++;
1001 		}
1002 	}
1003 
1004 	/* put it in and deallocate the old rrset */
1005 	rrset->data = d;
1006 	free(old);
1007 
1008 	/* now make rrsig set smaller */
1009 	if(sigold->count+sigold->rrsig_count == sigs) {
1010 		/* remove all sigs from rrsig, remove it entirely */
1011 		domain_remove_rrset(node, LDNS_RR_TYPE_RRSIG);
1012 		return 1;
1013 	}
1014 	log_assert(packed_rrset_sizeof(sigold) > sigs*(sizeof(size_t) +
1015 		sizeof(uint8_t*) + sizeof(time_t)) + sigsz);
1016 	sigd = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(sigold)
1017 		- sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
1018 		- sigsz);
1019 	if(!sigd) {
1020 		/* no need to free up d, it has already been placed in the
1021 		 * node->rrset structure */
1022 		log_err("out of memory");
1023 		return 0;
1024 	}
1025 	/* copy base values */
1026 	memcpy(sigd, sigold, sizeof(struct packed_rrset_data));
1027 	/* in sigd the RRSIGs are stored in the base of the RR, in count */
1028 	sigd->count -= sigs;
1029 	/* setup rr_len */
1030 	sigd->rr_len = (size_t*)((uint8_t*)sigd +
1031 		sizeof(struct packed_rrset_data));
1032 	j = 0;
1033 	for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1034 		if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1035 			sigold->rr_len[i]) != rr_type) {
1036 			sigd->rr_len[j] = sigold->rr_len[i];
1037 			j++;
1038 		}
1039 	}
1040 	packed_rrset_ptr_fixup(sigd);
1041 
1042 	/* copy old values into new rrsig array */
1043 	j = 0;
1044 	for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1045 		if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1046 			sigold->rr_len[i]) != rr_type) {
1047 			/* move this one over to location j */
1048 			sigd->rr_ttl[j] = sigold->rr_ttl[i];
1049 			memmove(sigd->rr_data[j], sigold->rr_data[i],
1050 				sigold->rr_len[i]);
1051 			if(j==0) sigd->ttl = sigd->rr_ttl[j];
1052 			else {
1053 				if(sigd->rr_ttl[j] < sigd->ttl)
1054 					sigd->ttl = sigd->rr_ttl[j];
1055 			}
1056 			j++;
1057 		}
1058 	}
1059 
1060 	/* put it in and deallocate the old rrset */
1061 	rrsig->data = sigd;
1062 	free(sigold);
1063 
1064 	return 1;
1065 }
1066 
1067 /** copy the rrsigs from the rrset to the rrsig rrset, because the rrset
1068  * is going to be deleted.  reallocates the RRSIG rrset data. */
1069 static int
1070 rrsigs_copy_from_rrset_to_rrsigset(struct auth_rrset* rrset,
1071 	struct auth_rrset* rrsigset)
1072 {
1073 	size_t i;
1074 	if(rrset->data->rrsig_count == 0)
1075 		return 1;
1076 
1077 	/* move them over one by one, because there might be duplicates,
1078 	 * duplicates are ignored */
1079 	for(i=rrset->data->count;
1080 		i<rrset->data->count+rrset->data->rrsig_count; i++) {
1081 		uint8_t* rdata = rrset->data->rr_data[i];
1082 		size_t rdatalen = rrset->data->rr_len[i];
1083 		time_t rr_ttl  = rrset->data->rr_ttl[i];
1084 
1085 		if(rdata_duplicate(rrsigset->data, rdata, rdatalen)) {
1086 			continue;
1087 		}
1088 		if(!rrset_add_rr(rrsigset, rr_ttl, rdata, rdatalen, 0))
1089 			return 0;
1090 	}
1091 	return 1;
1092 }
1093 
1094 /** Add rr to node, ignores duplicate RRs,
1095  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1096 static int
1097 az_domain_add_rr(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
1098 	uint8_t* rdata, size_t rdatalen, int* duplicate)
1099 {
1100 	struct auth_rrset* rrset;
1101 	/* packed rrsets have their rrsigs along with them, sort them out */
1102 	if(rr_type == LDNS_RR_TYPE_RRSIG) {
1103 		uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1104 		if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1105 			/* a node of the correct type exists, add the RRSIG
1106 			 * to the rrset of the covered data type */
1107 			if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1108 				if(duplicate) *duplicate = 1;
1109 				return 1;
1110 			}
1111 			if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 1))
1112 				return 0;
1113 		} else if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1114 			/* add RRSIG to rrset of type RRSIG */
1115 			if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1116 				if(duplicate) *duplicate = 1;
1117 				return 1;
1118 			}
1119 			if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1120 				return 0;
1121 		} else {
1122 			/* create rrset of type RRSIG */
1123 			if(!rrset_create(node, rr_type, rr_ttl, rdata,
1124 				rdatalen))
1125 				return 0;
1126 		}
1127 	} else {
1128 		/* normal RR type */
1129 		if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1130 			/* add data to existing node with data type */
1131 			if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1132 				if(duplicate) *duplicate = 1;
1133 				return 1;
1134 			}
1135 			if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1136 				return 0;
1137 		} else {
1138 			struct auth_rrset* rrsig;
1139 			/* create new node with data type */
1140 			if(!(rrset=rrset_create(node, rr_type, rr_ttl, rdata,
1141 				rdatalen)))
1142 				return 0;
1143 
1144 			/* see if node of type RRSIG has signatures that
1145 			 * cover the data type, and move them over */
1146 			/* and then make the RRSIG type smaller */
1147 			if((rrsig=az_domain_rrset(node, LDNS_RR_TYPE_RRSIG))
1148 				!= NULL) {
1149 				if(!rrset_moveover_rrsigs(node, rr_type,
1150 					rrset, rrsig))
1151 					return 0;
1152 			}
1153 		}
1154 	}
1155 	return 1;
1156 }
1157 
1158 /** insert RR into zone, ignore duplicates */
1159 static int
1160 az_insert_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1161 	size_t dname_len, int* duplicate)
1162 {
1163 	struct auth_data* node;
1164 	uint8_t* dname = rr;
1165 	uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1166 	uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1167 	uint32_t rr_ttl = sldns_wirerr_get_ttl(rr, rr_len, dname_len);
1168 	size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1169 		dname_len))+2;
1170 	/* rdata points to rdata prefixed with uint16 rdatalength */
1171 	uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1172 
1173 	if(rr_class != z->dclass) {
1174 		log_err("wrong class for RR");
1175 		return 0;
1176 	}
1177 	if(!(node=az_domain_find_or_create(z, dname, dname_len))) {
1178 		log_err("cannot create domain");
1179 		return 0;
1180 	}
1181 	if(!az_domain_add_rr(node, rr_type, rr_ttl, rdata, rdatalen,
1182 		duplicate)) {
1183 		log_err("cannot add RR to domain");
1184 		return 0;
1185 	}
1186 	if(z->rpz) {
1187 		if(!(rpz_insert_rr(z->rpz, z->name, z->namelen, dname,
1188 			dname_len, rr_type, rr_class, rr_ttl, rdata, rdatalen,
1189 			rr, rr_len)))
1190 			return 0;
1191 	}
1192 	return 1;
1193 }
1194 
1195 /** Remove rr from node, ignores nonexisting RRs,
1196  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1197 static int
1198 az_domain_remove_rr(struct auth_data* node, uint16_t rr_type,
1199 	uint8_t* rdata, size_t rdatalen, int* nonexist)
1200 {
1201 	struct auth_rrset* rrset;
1202 	size_t index = 0;
1203 
1204 	/* find the plain RR of the given type */
1205 	if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1206 		if(packed_rrset_find_rr(rrset->data, rdata, rdatalen, &index)) {
1207 			if(rrset->data->count == 1 &&
1208 				rrset->data->rrsig_count == 0) {
1209 				/* last RR, delete the rrset */
1210 				domain_remove_rrset(node, rr_type);
1211 			} else if(rrset->data->count == 1 &&
1212 				rrset->data->rrsig_count != 0) {
1213 				/* move RRSIGs to the RRSIG rrset, or
1214 				 * this one becomes that RRset */
1215 				struct auth_rrset* rrsigset = az_domain_rrset(
1216 					node, LDNS_RR_TYPE_RRSIG);
1217 				if(rrsigset) {
1218 					/* move left over rrsigs to the
1219 					 * existing rrset of type RRSIG */
1220 					rrsigs_copy_from_rrset_to_rrsigset(
1221 						rrset, rrsigset);
1222 					/* and then delete the rrset */
1223 					domain_remove_rrset(node, rr_type);
1224 				} else {
1225 					/* no rrset of type RRSIG, this
1226 					 * set is now of that type,
1227 					 * just remove the rr */
1228 					if(!rrset_remove_rr(rrset, index))
1229 						return 0;
1230 					rrset->type = LDNS_RR_TYPE_RRSIG;
1231 					rrset->data->count = rrset->data->rrsig_count;
1232 					rrset->data->rrsig_count = 0;
1233 				}
1234 			} else {
1235 				/* remove the RR from the rrset */
1236 				if(!rrset_remove_rr(rrset, index))
1237 					return 0;
1238 			}
1239 			return 1;
1240 		}
1241 		/* rr not found in rrset */
1242 	}
1243 
1244 	/* is it a type RRSIG, look under the covered type */
1245 	if(rr_type == LDNS_RR_TYPE_RRSIG) {
1246 		uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1247 		if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1248 			if(az_rrset_find_rrsig(rrset->data, rdata, rdatalen,
1249 				&index)) {
1250 				/* rrsig should have d->count > 0, be
1251 				 * over some rr of that type */
1252 				/* remove the rrsig from the rrsigs list of the
1253 				 * rrset */
1254 				if(!rrset_remove_rr(rrset, index))
1255 					return 0;
1256 				return 1;
1257 			}
1258 		}
1259 		/* also RRSIG not found */
1260 	}
1261 
1262 	/* nothing found to delete */
1263 	if(nonexist) *nonexist = 1;
1264 	return 1;
1265 }
1266 
1267 /** remove RR from zone, ignore if it does not exist, false on alloc failure*/
1268 static int
1269 az_remove_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1270 	size_t dname_len, int* nonexist)
1271 {
1272 	struct auth_data* node;
1273 	uint8_t* dname = rr;
1274 	uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1275 	uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1276 	size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1277 		dname_len))+2;
1278 	/* rdata points to rdata prefixed with uint16 rdatalength */
1279 	uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1280 
1281 	if(rr_class != z->dclass) {
1282 		log_err("wrong class for RR");
1283 		/* really also a nonexisting entry, because no records
1284 		 * of that class in the zone, but return an error because
1285 		 * getting records of the wrong class is a failure of the
1286 		 * zone transfer */
1287 		return 0;
1288 	}
1289 	node = az_find_name(z, dname, dname_len);
1290 	if(!node) {
1291 		/* node with that name does not exist */
1292 		/* nonexisting entry, because no such name */
1293 		*nonexist = 1;
1294 		return 1;
1295 	}
1296 	if(!az_domain_remove_rr(node, rr_type, rdata, rdatalen, nonexist)) {
1297 		/* alloc failure or so */
1298 		return 0;
1299 	}
1300 	/* remove the node, if necessary */
1301 	/* an rrsets==NULL entry is not kept around for empty nonterminals,
1302 	 * and also parent nodes are not kept around, so we just delete it */
1303 	if(node->rrsets == NULL) {
1304 		(void)rbtree_delete(&z->data, node);
1305 		auth_data_delete(node);
1306 	}
1307 	if(z->rpz) {
1308 		rpz_remove_rr(z->rpz, z->namelen, dname, dname_len, rr_type,
1309 			rr_class, rdata, rdatalen);
1310 	}
1311 	return 1;
1312 }
1313 
1314 /** decompress an RR into the buffer where it'll be an uncompressed RR
1315  * with uncompressed dname and uncompressed rdata (dnames) */
1316 static int
1317 decompress_rr_into_buffer(struct sldns_buffer* buf, uint8_t* pkt,
1318 	size_t pktlen, uint8_t* dname, uint16_t rr_type, uint16_t rr_class,
1319 	uint32_t rr_ttl, uint8_t* rr_data, uint16_t rr_rdlen)
1320 {
1321 	sldns_buffer pktbuf;
1322 	size_t dname_len = 0;
1323 	size_t rdlenpos;
1324 	size_t rdlen;
1325 	uint8_t* rd;
1326 	const sldns_rr_descriptor* desc;
1327 	sldns_buffer_init_frm_data(&pktbuf, pkt, pktlen);
1328 	sldns_buffer_clear(buf);
1329 
1330 	/* decompress dname */
1331 	sldns_buffer_set_position(&pktbuf,
1332 		(size_t)(dname - sldns_buffer_current(&pktbuf)));
1333 	dname_len = pkt_dname_len(&pktbuf);
1334 	if(dname_len == 0) return 0; /* parse fail on dname */
1335 	if(!sldns_buffer_available(buf, dname_len)) return 0;
1336 	dname_pkt_copy(&pktbuf, sldns_buffer_current(buf), dname);
1337 	sldns_buffer_skip(buf, (ssize_t)dname_len);
1338 
1339 	/* type, class, ttl and rdatalength fields */
1340 	if(!sldns_buffer_available(buf, 10)) return 0;
1341 	sldns_buffer_write_u16(buf, rr_type);
1342 	sldns_buffer_write_u16(buf, rr_class);
1343 	sldns_buffer_write_u32(buf, rr_ttl);
1344 	rdlenpos = sldns_buffer_position(buf);
1345 	sldns_buffer_write_u16(buf, 0); /* rd length position */
1346 
1347 	/* decompress rdata */
1348 	desc = sldns_rr_descript(rr_type);
1349 	rd = rr_data;
1350 	rdlen = rr_rdlen;
1351 	if(rdlen > 0 && desc && desc->_dname_count > 0) {
1352 		int count = (int)desc->_dname_count;
1353 		int rdf = 0;
1354 		size_t len; /* how much rdata to plain copy */
1355 		size_t uncompressed_len, compressed_len;
1356 		size_t oldpos;
1357 		/* decompress dnames. */
1358 		while(rdlen > 0 && count) {
1359 			switch(desc->_wireformat[rdf]) {
1360 			case LDNS_RDF_TYPE_DNAME:
1361 				sldns_buffer_set_position(&pktbuf,
1362 					(size_t)(rd -
1363 					sldns_buffer_begin(&pktbuf)));
1364 				oldpos = sldns_buffer_position(&pktbuf);
1365 				/* moves pktbuf to right after the
1366 				 * compressed dname, and returns uncompressed
1367 				 * dname length */
1368 				uncompressed_len = pkt_dname_len(&pktbuf);
1369 				if(!uncompressed_len)
1370 					return 0; /* parse error in dname */
1371 				if(!sldns_buffer_available(buf,
1372 					uncompressed_len))
1373 					/* dname too long for buffer */
1374 					return 0;
1375 				dname_pkt_copy(&pktbuf,
1376 					sldns_buffer_current(buf), rd);
1377 				sldns_buffer_skip(buf, (ssize_t)uncompressed_len);
1378 				compressed_len = sldns_buffer_position(
1379 					&pktbuf) - oldpos;
1380 				rd += compressed_len;
1381 				rdlen -= compressed_len;
1382 				count--;
1383 				len = 0;
1384 				break;
1385 			case LDNS_RDF_TYPE_STR:
1386 				len = rd[0] + 1;
1387 				break;
1388 			default:
1389 				len = get_rdf_size(desc->_wireformat[rdf]);
1390 				break;
1391 			}
1392 			if(len) {
1393 				if(!sldns_buffer_available(buf, len))
1394 					return 0; /* too long for buffer */
1395 				sldns_buffer_write(buf, rd, len);
1396 				rd += len;
1397 				rdlen -= len;
1398 			}
1399 			rdf++;
1400 		}
1401 	}
1402 	/* copy remaining data */
1403 	if(rdlen > 0) {
1404 		if(!sldns_buffer_available(buf, rdlen)) return 0;
1405 		sldns_buffer_write(buf, rd, rdlen);
1406 	}
1407 	/* fixup rdlength */
1408 	sldns_buffer_write_u16_at(buf, rdlenpos,
1409 		sldns_buffer_position(buf)-rdlenpos-2);
1410 	sldns_buffer_flip(buf);
1411 	return 1;
1412 }
1413 
1414 /** insert RR into zone, from packet, decompress RR,
1415  * if duplicate is nonNULL set the flag but otherwise ignore duplicates */
1416 static int
1417 az_insert_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1418 	struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1419 	uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1420 	uint16_t rr_rdlen, int* duplicate)
1421 {
1422 	uint8_t* rr;
1423 	size_t rr_len;
1424 	size_t dname_len;
1425 	if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1426 		rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1427 		log_err("could not decompress RR");
1428 		return 0;
1429 	}
1430 	rr = sldns_buffer_begin(scratch_buffer);
1431 	rr_len = sldns_buffer_limit(scratch_buffer);
1432 	dname_len = dname_valid(rr, rr_len);
1433 	return az_insert_rr(z, rr, rr_len, dname_len, duplicate);
1434 }
1435 
1436 /** remove RR from zone, from packet, decompress RR,
1437  * if nonexist is nonNULL set the flag but otherwise ignore nonexisting entries*/
1438 static int
1439 az_remove_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1440 	struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1441 	uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1442 	uint16_t rr_rdlen, int* nonexist)
1443 {
1444 	uint8_t* rr;
1445 	size_t rr_len;
1446 	size_t dname_len;
1447 	if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1448 		rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1449 		log_err("could not decompress RR");
1450 		return 0;
1451 	}
1452 	rr = sldns_buffer_begin(scratch_buffer);
1453 	rr_len = sldns_buffer_limit(scratch_buffer);
1454 	dname_len = dname_valid(rr, rr_len);
1455 	return az_remove_rr(z, rr, rr_len, dname_len, nonexist);
1456 }
1457 
1458 /**
1459  * Parse zonefile
1460  * @param z: zone to read in.
1461  * @param in: file to read from (just opened).
1462  * @param rr: buffer to use for RRs, 64k.
1463  *	passed so that recursive includes can use the same buffer and do
1464  *	not grow the stack too much.
1465  * @param rrbuflen: sizeof rr buffer.
1466  * @param state: parse state with $ORIGIN, $TTL and 'prev-dname' and so on,
1467  *	that is kept between includes.
1468  *	The lineno is set at 1 and then increased by the function.
1469  * @param fname: file name.
1470  * @param depth: recursion depth for includes
1471  * @param cfg: config for chroot.
1472  * returns false on failure, has printed an error message
1473  */
1474 static int
1475 az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
1476 	struct sldns_file_parse_state* state, char* fname, int depth,
1477 	struct config_file* cfg)
1478 {
1479 	size_t rr_len, dname_len;
1480 	int status;
1481 	state->lineno = 1;
1482 
1483 	while(!feof(in)) {
1484 		rr_len = rrbuflen;
1485 		dname_len = 0;
1486 		status = sldns_fp2wire_rr_buf(in, rr, &rr_len, &dname_len,
1487 			state);
1488 		if(status == LDNS_WIREPARSE_ERR_INCLUDE && rr_len == 0) {
1489 			/* we have $INCLUDE or $something */
1490 			if(strncmp((char*)rr, "$INCLUDE ", 9) == 0 ||
1491 			   strncmp((char*)rr, "$INCLUDE\t", 9) == 0) {
1492 				FILE* inc;
1493 				int lineno_orig = state->lineno;
1494 				char* incfile = (char*)rr + 8;
1495 				if(depth > MAX_INCLUDE_DEPTH) {
1496 					log_err("%s:%d max include depth"
1497 					  "exceeded", fname, state->lineno);
1498 					return 0;
1499 				}
1500 				/* skip spaces */
1501 				while(*incfile == ' ' || *incfile == '\t')
1502 					incfile++;
1503 				/* adjust for chroot on include file */
1504 				if(cfg->chrootdir && cfg->chrootdir[0] &&
1505 					strncmp(incfile, cfg->chrootdir,
1506 						strlen(cfg->chrootdir)) == 0)
1507 					incfile += strlen(cfg->chrootdir);
1508 				incfile = strdup(incfile);
1509 				if(!incfile) {
1510 					log_err("malloc failure");
1511 					return 0;
1512 				}
1513 				verbose(VERB_ALGO, "opening $INCLUDE %s",
1514 					incfile);
1515 				inc = fopen(incfile, "r");
1516 				if(!inc) {
1517 					log_err("%s:%d cannot open include "
1518 						"file %s: %s", fname,
1519 						lineno_orig, incfile,
1520 						strerror(errno));
1521 					free(incfile);
1522 					return 0;
1523 				}
1524 				/* recurse read that file now */
1525 				if(!az_parse_file(z, inc, rr, rrbuflen,
1526 					state, incfile, depth+1, cfg)) {
1527 					log_err("%s:%d cannot parse include "
1528 						"file %s", fname,
1529 						lineno_orig, incfile);
1530 					fclose(inc);
1531 					free(incfile);
1532 					return 0;
1533 				}
1534 				fclose(inc);
1535 				verbose(VERB_ALGO, "done with $INCLUDE %s",
1536 					incfile);
1537 				free(incfile);
1538 				state->lineno = lineno_orig;
1539 			}
1540 			continue;
1541 		}
1542 		if(status != 0) {
1543 			log_err("parse error %s %d:%d: %s", fname,
1544 				state->lineno, LDNS_WIREPARSE_OFFSET(status),
1545 				sldns_get_errorstr_parse(status));
1546 			return 0;
1547 		}
1548 		if(rr_len == 0) {
1549 			/* EMPTY line, TTL or ORIGIN */
1550 			continue;
1551 		}
1552 		/* insert wirerr in rrbuf */
1553 		if(!az_insert_rr(z, rr, rr_len, dname_len, NULL)) {
1554 			char buf[17];
1555 			sldns_wire2str_type_buf(sldns_wirerr_get_type(rr,
1556 				rr_len, dname_len), buf, sizeof(buf));
1557 			log_err("%s:%d cannot insert RR of type %s",
1558 				fname, state->lineno, buf);
1559 			return 0;
1560 		}
1561 	}
1562 	return 1;
1563 }
1564 
1565 int
1566 auth_zone_read_zonefile(struct auth_zone* z, struct config_file* cfg)
1567 {
1568 	uint8_t rr[LDNS_RR_BUF_SIZE];
1569 	struct sldns_file_parse_state state;
1570 	char* zfilename;
1571 	FILE* in;
1572 	if(!z || !z->zonefile || z->zonefile[0]==0)
1573 		return 1; /* no file, or "", nothing to read */
1574 
1575 	zfilename = z->zonefile;
1576 	if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
1577 		cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
1578 		zfilename += strlen(cfg->chrootdir);
1579 	if(verbosity >= VERB_ALGO) {
1580 		char nm[255+1];
1581 		dname_str(z->name, nm);
1582 		verbose(VERB_ALGO, "read zonefile %s for %s", zfilename, nm);
1583 	}
1584 	in = fopen(zfilename, "r");
1585 	if(!in) {
1586 		char* n = sldns_wire2str_dname(z->name, z->namelen);
1587 		if(z->zone_is_slave && errno == ENOENT) {
1588 			/* we fetch the zone contents later, no file yet */
1589 			verbose(VERB_ALGO, "no zonefile %s for %s",
1590 				zfilename, n?n:"error");
1591 			free(n);
1592 			return 1;
1593 		}
1594 		log_err("cannot open zonefile %s for %s: %s",
1595 			zfilename, n?n:"error", strerror(errno));
1596 		free(n);
1597 		return 0;
1598 	}
1599 
1600 	/* clear the data tree */
1601 	traverse_postorder(&z->data, auth_data_del, NULL);
1602 	rbtree_init(&z->data, &auth_data_cmp);
1603 	/* clear the RPZ policies */
1604 	if(z->rpz)
1605 		rpz_clear(z->rpz);
1606 
1607 	memset(&state, 0, sizeof(state));
1608 	/* default TTL to 3600 */
1609 	state.default_ttl = 3600;
1610 	/* set $ORIGIN to the zone name */
1611 	if(z->namelen <= sizeof(state.origin)) {
1612 		memcpy(state.origin, z->name, z->namelen);
1613 		state.origin_len = z->namelen;
1614 	}
1615 	/* parse the (toplevel) file */
1616 	if(!az_parse_file(z, in, rr, sizeof(rr), &state, zfilename, 0, cfg)) {
1617 		char* n = sldns_wire2str_dname(z->name, z->namelen);
1618 		log_err("error parsing zonefile %s for %s",
1619 			zfilename, n?n:"error");
1620 		free(n);
1621 		fclose(in);
1622 		return 0;
1623 	}
1624 	fclose(in);
1625 
1626 	if(z->rpz)
1627 		rpz_finish_config(z->rpz);
1628 	return 1;
1629 }
1630 
1631 /** write buffer to file and check return codes */
1632 static int
1633 write_out(FILE* out, const char* str, size_t len)
1634 {
1635 	size_t r;
1636 	if(len == 0)
1637 		return 1;
1638 	r = fwrite(str, 1, len, out);
1639 	if(r == 0) {
1640 		log_err("write failed: %s", strerror(errno));
1641 		return 0;
1642 	} else if(r < len) {
1643 		log_err("write failed: too short (disk full?)");
1644 		return 0;
1645 	}
1646 	return 1;
1647 }
1648 
1649 /** convert auth rr to string */
1650 static int
1651 auth_rr_to_string(uint8_t* nm, size_t nmlen, uint16_t tp, uint16_t cl,
1652 	struct packed_rrset_data* data, size_t i, char* s, size_t buflen)
1653 {
1654 	int w = 0;
1655 	size_t slen = buflen, datlen;
1656 	uint8_t* dat;
1657 	if(i >= data->count) tp = LDNS_RR_TYPE_RRSIG;
1658 	dat = nm;
1659 	datlen = nmlen;
1660 	w += sldns_wire2str_dname_scan(&dat, &datlen, &s, &slen, NULL, 0, NULL);
1661 	w += sldns_str_print(&s, &slen, "\t");
1662 	w += sldns_str_print(&s, &slen, "%lu\t", (unsigned long)data->rr_ttl[i]);
1663 	w += sldns_wire2str_class_print(&s, &slen, cl);
1664 	w += sldns_str_print(&s, &slen, "\t");
1665 	w += sldns_wire2str_type_print(&s, &slen, tp);
1666 	w += sldns_str_print(&s, &slen, "\t");
1667 	datlen = data->rr_len[i]-2;
1668 	dat = data->rr_data[i]+2;
1669 	w += sldns_wire2str_rdata_scan(&dat, &datlen, &s, &slen, tp, NULL, 0, NULL);
1670 
1671 	if(tp == LDNS_RR_TYPE_DNSKEY) {
1672 		w += sldns_str_print(&s, &slen, " ;{id = %u}",
1673 			sldns_calc_keytag_raw(data->rr_data[i]+2,
1674 				data->rr_len[i]-2));
1675 	}
1676 	w += sldns_str_print(&s, &slen, "\n");
1677 
1678 	if(w >= (int)buflen) {
1679 		log_nametypeclass(NO_VERBOSE, "RR too long to print", nm, tp, cl);
1680 		return 0;
1681 	}
1682 	return 1;
1683 }
1684 
1685 /** write rrset to file */
1686 static int
1687 auth_zone_write_rrset(struct auth_zone* z, struct auth_data* node,
1688 	struct auth_rrset* r, FILE* out)
1689 {
1690 	size_t i, count = r->data->count + r->data->rrsig_count;
1691 	char buf[LDNS_RR_BUF_SIZE];
1692 	for(i=0; i<count; i++) {
1693 		if(!auth_rr_to_string(node->name, node->namelen, r->type,
1694 			z->dclass, r->data, i, buf, sizeof(buf))) {
1695 			verbose(VERB_ALGO, "failed to rr2str rr %d", (int)i);
1696 			continue;
1697 		}
1698 		if(!write_out(out, buf, strlen(buf)))
1699 			return 0;
1700 	}
1701 	return 1;
1702 }
1703 
1704 /** write domain to file */
1705 static int
1706 auth_zone_write_domain(struct auth_zone* z, struct auth_data* n, FILE* out)
1707 {
1708 	struct auth_rrset* r;
1709 	/* if this is zone apex, write SOA first */
1710 	if(z->namelen == n->namelen) {
1711 		struct auth_rrset* soa = az_domain_rrset(n, LDNS_RR_TYPE_SOA);
1712 		if(soa) {
1713 			if(!auth_zone_write_rrset(z, n, soa, out))
1714 				return 0;
1715 		}
1716 	}
1717 	/* write all the RRsets for this domain */
1718 	for(r = n->rrsets; r; r = r->next) {
1719 		if(z->namelen == n->namelen &&
1720 			r->type == LDNS_RR_TYPE_SOA)
1721 			continue; /* skip SOA here */
1722 		if(!auth_zone_write_rrset(z, n, r, out))
1723 			return 0;
1724 	}
1725 	return 1;
1726 }
1727 
1728 int auth_zone_write_file(struct auth_zone* z, const char* fname)
1729 {
1730 	FILE* out;
1731 	struct auth_data* n;
1732 	out = fopen(fname, "w");
1733 	if(!out) {
1734 		log_err("could not open %s: %s", fname, strerror(errno));
1735 		return 0;
1736 	}
1737 	RBTREE_FOR(n, struct auth_data*, &z->data) {
1738 		if(!auth_zone_write_domain(z, n, out)) {
1739 			log_err("could not write domain to %s", fname);
1740 			fclose(out);
1741 			return 0;
1742 		}
1743 	}
1744 	fclose(out);
1745 	return 1;
1746 }
1747 
1748 /** offline verify for zonemd, while reading a zone file to immediately
1749  * spot bad hashes in zonefile as they are read.
1750  * Creates temp buffers, but uses anchors and validation environment
1751  * from the module_env. */
1752 static void
1753 zonemd_offline_verify(struct auth_zone* z, struct module_env* env_for_val,
1754 	struct module_stack* mods)
1755 {
1756 	struct module_env env;
1757 	time_t now = 0;
1758 	if(!z->zonemd_check)
1759 		return;
1760 	env = *env_for_val;
1761 	env.scratch_buffer = sldns_buffer_new(env.cfg->msg_buffer_size);
1762 	if(!env.scratch_buffer) {
1763 		log_err("out of memory");
1764 		goto clean_exit;
1765 	}
1766 	env.scratch = regional_create();
1767 	if(!env.now) {
1768 		env.now = &now;
1769 		now = time(NULL);
1770 	}
1771 	if(!env.scratch) {
1772 		log_err("out of memory");
1773 		goto clean_exit;
1774 	}
1775 	auth_zone_verify_zonemd(z, &env, mods, NULL, 1, 0);
1776 
1777 clean_exit:
1778 	/* clean up and exit */
1779 	sldns_buffer_free(env.scratch_buffer);
1780 	regional_destroy(env.scratch);
1781 }
1782 
1783 /** read all auth zones from file (if they have) */
1784 static int
1785 auth_zones_read_zones(struct auth_zones* az, struct config_file* cfg,
1786 	struct module_env* env, struct module_stack* mods)
1787 {
1788 	struct auth_zone* z;
1789 	lock_rw_wrlock(&az->lock);
1790 	RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
1791 		lock_rw_wrlock(&z->lock);
1792 		if(!auth_zone_read_zonefile(z, cfg)) {
1793 			lock_rw_unlock(&z->lock);
1794 			lock_rw_unlock(&az->lock);
1795 			return 0;
1796 		}
1797 		if(z->zonefile && z->zonefile[0]!=0 && env)
1798 			zonemd_offline_verify(z, env, mods);
1799 		lock_rw_unlock(&z->lock);
1800 	}
1801 	lock_rw_unlock(&az->lock);
1802 	return 1;
1803 }
1804 
1805 /** fetch the content of a ZONEMD RR from the rdata */
1806 static int zonemd_fetch_parameters(struct auth_rrset* zonemd_rrset, size_t i,
1807 	uint32_t* serial, int* scheme, int* hashalgo, uint8_t** hash,
1808 	size_t* hashlen)
1809 {
1810 	size_t rr_len;
1811 	uint8_t* rdata;
1812 	if(i >= zonemd_rrset->data->count)
1813 		return 0;
1814 	rr_len = zonemd_rrset->data->rr_len[i];
1815 	if(rr_len < 2+4+1+1)
1816 		return 0; /* too short, for rdlen+serial+scheme+algo */
1817 	rdata = zonemd_rrset->data->rr_data[i];
1818 	*serial = sldns_read_uint32(rdata+2);
1819 	*scheme = rdata[6];
1820 	*hashalgo = rdata[7];
1821 	*hashlen = rr_len - 8;
1822 	if(*hashlen == 0)
1823 		*hash = NULL;
1824 	else	*hash = rdata+8;
1825 	return 1;
1826 }
1827 
1828 /**
1829  * See if the ZONEMD scheme, hash occurs more than once.
1830  * @param zonemd_rrset: the zonemd rrset to check with the RRs in it.
1831  * @param index: index of the original, this is allowed to have that
1832  * 	scheme and hashalgo, but other RRs should not have it.
1833  * @param scheme: the scheme to check for.
1834  * @param hashalgo: the hash algorithm to check for.
1835  * @return true if it occurs more than once.
1836  */
1837 static int zonemd_is_duplicate_scheme_hash(struct auth_rrset* zonemd_rrset,
1838 	size_t index, int scheme, int hashalgo)
1839 {
1840 	size_t j;
1841 	for(j=0; j<zonemd_rrset->data->count; j++) {
1842 		uint32_t serial2 = 0;
1843 		int scheme2 = 0, hashalgo2 = 0;
1844 		uint8_t* hash2 = NULL;
1845 		size_t hashlen2 = 0;
1846 		if(index == j) {
1847 			/* this is the original */
1848 			continue;
1849 		}
1850 		if(!zonemd_fetch_parameters(zonemd_rrset, j, &serial2,
1851 			&scheme2, &hashalgo2, &hash2, &hashlen2)) {
1852 			/* malformed, skip it */
1853 			continue;
1854 		}
1855 		if(scheme == scheme2 && hashalgo == hashalgo2) {
1856 			/* duplicate scheme, hash */
1857 			verbose(VERB_ALGO, "zonemd duplicate for scheme %d "
1858 				"and hash %d", scheme, hashalgo);
1859 			return 1;
1860 		}
1861 	}
1862 	return 0;
1863 }
1864 
1865 /**
1866  * Check ZONEMDs if present for the auth zone.  Depending on config
1867  * it can warn or fail on that.  Checks the hash of the ZONEMD.
1868  * @param z: auth zone to check for.
1869  * 	caller must hold lock on zone.
1870  * @param env: module env for temp buffers.
1871  * @param reason: returned on failure.
1872  * @return false on failure, true if hash checks out.
1873  */
1874 static int auth_zone_zonemd_check_hash(struct auth_zone* z,
1875 	struct module_env* env, char** reason)
1876 {
1877 	/* loop over ZONEMDs and see which one is valid. if not print
1878 	 * failure (depending on config) */
1879 	struct auth_data* apex;
1880 	struct auth_rrset* zonemd_rrset;
1881 	size_t i;
1882 	struct regional* region = NULL;
1883 	struct sldns_buffer* buf = NULL;
1884 	uint32_t soa_serial = 0;
1885 	region = env->scratch;
1886 	regional_free_all(region);
1887 	buf = env->scratch_buffer;
1888 	if(!auth_zone_get_serial(z, &soa_serial)) {
1889 		*reason = "zone has no SOA serial";
1890 		return 0;
1891 	}
1892 
1893 	apex = az_find_name(z, z->name, z->namelen);
1894 	if(!apex) {
1895 		*reason = "zone has no apex";
1896 		return 0;
1897 	}
1898 	zonemd_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_ZONEMD);
1899 	if(!zonemd_rrset || zonemd_rrset->data->count==0) {
1900 		*reason = "zone has no ZONEMD";
1901 		return 0; /* no RRset or no RRs in rrset */
1902 	}
1903 
1904 	/* we have a ZONEMD, check if it is correct */
1905 	for(i=0; i<zonemd_rrset->data->count; i++) {
1906 		uint32_t serial = 0;
1907 		int scheme = 0, hashalgo = 0;
1908 		uint8_t* hash = NULL;
1909 		size_t hashlen = 0;
1910 		if(!zonemd_fetch_parameters(zonemd_rrset, i, &serial, &scheme,
1911 			&hashalgo, &hash, &hashlen)) {
1912 			/* malformed RR */
1913 			*reason = "ZONEMD rdata malformed";
1914 			continue;
1915 		}
1916 		/* check for duplicates */
1917 		if(zonemd_is_duplicate_scheme_hash(zonemd_rrset, i, scheme,
1918 			hashalgo)) {
1919 			/* duplicate hash of the same scheme,hash
1920 			 * is not allowed. */
1921 			*reason = "ZONEMD RRSet contains more than one RR "
1922 				"with the same scheme and hash algorithm";
1923 			continue;
1924 		}
1925 		regional_free_all(region);
1926 		if(serial != soa_serial) {
1927 			*reason = "ZONEMD serial is wrong";
1928 			continue;
1929 		}
1930 		if(auth_zone_generate_zonemd_check(z, scheme, hashalgo,
1931 			hash, hashlen, region, buf, reason)) {
1932 			/* success */
1933 			if(verbosity >= VERB_ALGO) {
1934 				char zstr[255+1];
1935 				dname_str(z->name, zstr);
1936 				verbose(VERB_ALGO, "auth-zone %s ZONEMD hash is correct", zstr);
1937 			}
1938 			return 1;
1939 		}
1940 		/* try next one */
1941 	}
1942 	/* fail, we may have reason */
1943 	if(!*reason)
1944 		*reason = "no ZONEMD records found";
1945 	if(verbosity >= VERB_ALGO) {
1946 		char zstr[255+1];
1947 		dname_str(z->name, zstr);
1948 		verbose(VERB_ALGO, "auth-zone %s ZONEMD failed: %s", zstr, *reason);
1949 	}
1950 	return 0;
1951 }
1952 
1953 /** find serial number of zone or false if none */
1954 int
1955 auth_zone_get_serial(struct auth_zone* z, uint32_t* serial)
1956 {
1957 	struct auth_data* apex;
1958 	struct auth_rrset* soa;
1959 	struct packed_rrset_data* d;
1960 	apex = az_find_name(z, z->name, z->namelen);
1961 	if(!apex) return 0;
1962 	soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1963 	if(!soa || soa->data->count==0)
1964 		return 0; /* no RRset or no RRs in rrset */
1965 	if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1966 	d = soa->data;
1967 	*serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
1968 	return 1;
1969 }
1970 
1971 /** Find auth_zone SOA and populate the values in xfr(soa values). */
1972 int
1973 xfr_find_soa(struct auth_zone* z, struct auth_xfer* xfr)
1974 {
1975 	struct auth_data* apex;
1976 	struct auth_rrset* soa;
1977 	struct packed_rrset_data* d;
1978 	apex = az_find_name(z, z->name, z->namelen);
1979 	if(!apex) return 0;
1980 	soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1981 	if(!soa || soa->data->count==0)
1982 		return 0; /* no RRset or no RRs in rrset */
1983 	if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1984 	/* SOA record ends with serial, refresh, retry, expiry, minimum,
1985 	 * as 4 byte fields */
1986 	d = soa->data;
1987 	xfr->have_zone = 1;
1988 	xfr->serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
1989 	xfr->refresh = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-16));
1990 	xfr->retry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-12));
1991 	xfr->expiry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-8));
1992 	/* soa minimum at d->rr_len[0]-4 */
1993 	return 1;
1994 }
1995 
1996 /**
1997  * Setup auth_xfer zone
1998  * This populates the have_zone, soa values, and so on times.
1999  * Doesn't do network traffic yet, can set option flags.
2000  * @param z: locked by caller, and modified for setup
2001  * @param x: locked by caller, and modified.
2002  * @return false on failure.
2003  */
2004 static int
2005 auth_xfer_setup(struct auth_zone* z, struct auth_xfer* x)
2006 {
2007 	/* for a zone without zone transfers, x==NULL, so skip them,
2008 	 * i.e. the zone config is fixed with no masters or urls */
2009 	if(!z || !x) return 1;
2010 	if(!xfr_find_soa(z, x)) {
2011 		return 1;
2012 	}
2013 	/* nothing for probe, nextprobe and transfer tasks */
2014 	return 1;
2015 }
2016 
2017 /**
2018  * Setup all zones
2019  * @param az: auth zones structure
2020  * @return false on failure.
2021  */
2022 static int
2023 auth_zones_setup_zones(struct auth_zones* az)
2024 {
2025 	struct auth_zone* z;
2026 	struct auth_xfer* x;
2027 	lock_rw_wrlock(&az->lock);
2028 	RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2029 		lock_rw_wrlock(&z->lock);
2030 		x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
2031 		if(x) {
2032 			lock_basic_lock(&x->lock);
2033 		}
2034 		if(!auth_xfer_setup(z, x)) {
2035 			if(x) {
2036 				lock_basic_unlock(&x->lock);
2037 			}
2038 			lock_rw_unlock(&z->lock);
2039 			lock_rw_unlock(&az->lock);
2040 			return 0;
2041 		}
2042 		if(x) {
2043 			lock_basic_unlock(&x->lock);
2044 		}
2045 		lock_rw_unlock(&z->lock);
2046 	}
2047 	lock_rw_unlock(&az->lock);
2048 	return 1;
2049 }
2050 
2051 /** set config items and create zones */
2052 static int
2053 auth_zones_cfg(struct auth_zones* az, struct config_auth* c)
2054 {
2055 	struct auth_zone* z;
2056 	struct auth_xfer* x = NULL;
2057 
2058 	/* create zone */
2059 	if(c->isrpz) {
2060 		/* if the rpz lock is needed, grab it before the other
2061 		 * locks to avoid a lock dependency cycle */
2062 		lock_rw_wrlock(&az->rpz_lock);
2063 	}
2064 	lock_rw_wrlock(&az->lock);
2065 	if(!(z=auth_zones_find_or_add_zone(az, c->name))) {
2066 		lock_rw_unlock(&az->lock);
2067 		if(c->isrpz) {
2068 			lock_rw_unlock(&az->rpz_lock);
2069 		}
2070 		return 0;
2071 	}
2072 	if(c->masters || c->urls) {
2073 		if(!(x=auth_zones_find_or_add_xfer(az, z))) {
2074 			lock_rw_unlock(&az->lock);
2075 			lock_rw_unlock(&z->lock);
2076 			if(c->isrpz) {
2077 				lock_rw_unlock(&az->rpz_lock);
2078 			}
2079 			return 0;
2080 		}
2081 	}
2082 	if(c->for_downstream)
2083 		az->have_downstream = 1;
2084 	lock_rw_unlock(&az->lock);
2085 
2086 	/* set options */
2087 	z->zone_deleted = 0;
2088 	if(!auth_zone_set_zonefile(z, c->zonefile)) {
2089 		if(x) {
2090 			lock_basic_unlock(&x->lock);
2091 		}
2092 		lock_rw_unlock(&z->lock);
2093 		if(c->isrpz) {
2094 			lock_rw_unlock(&az->rpz_lock);
2095 		}
2096 		return 0;
2097 	}
2098 	z->for_downstream = c->for_downstream;
2099 	z->for_upstream = c->for_upstream;
2100 	z->fallback_enabled = c->fallback_enabled;
2101 	z->zonemd_check = c->zonemd_check;
2102 	z->zonemd_reject_absence = c->zonemd_reject_absence;
2103 	if(c->isrpz && !z->rpz){
2104 		if(!(z->rpz = rpz_create(c))){
2105 			fatal_exit("Could not setup RPZ zones");
2106 			return 0;
2107 		}
2108 		lock_protect(&z->lock, &z->rpz->local_zones, sizeof(*z->rpz));
2109 		/* the az->rpz_lock is locked above */
2110 		z->rpz_az_next = az->rpz_first;
2111 		if(az->rpz_first)
2112 			az->rpz_first->rpz_az_prev = z;
2113 		az->rpz_first = z;
2114 	}
2115 	if(c->isrpz) {
2116 		lock_rw_unlock(&az->rpz_lock);
2117 	}
2118 
2119 	/* xfer zone */
2120 	if(x) {
2121 		z->zone_is_slave = 1;
2122 		/* set options on xfer zone */
2123 		if(!xfer_set_masters(&x->task_probe->masters, c, 0)) {
2124 			lock_basic_unlock(&x->lock);
2125 			lock_rw_unlock(&z->lock);
2126 			return 0;
2127 		}
2128 		if(!xfer_set_masters(&x->task_transfer->masters, c, 1)) {
2129 			lock_basic_unlock(&x->lock);
2130 			lock_rw_unlock(&z->lock);
2131 			return 0;
2132 		}
2133 		lock_basic_unlock(&x->lock);
2134 	}
2135 
2136 	lock_rw_unlock(&z->lock);
2137 	return 1;
2138 }
2139 
2140 /** set all auth zones deleted, then in auth_zones_cfg, it marks them
2141  * as nondeleted (if they are still in the config), and then later
2142  * we can find deleted zones */
2143 static void
2144 az_setall_deleted(struct auth_zones* az)
2145 {
2146 	struct auth_zone* z;
2147 	lock_rw_wrlock(&az->lock);
2148 	RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2149 		lock_rw_wrlock(&z->lock);
2150 		z->zone_deleted = 1;
2151 		lock_rw_unlock(&z->lock);
2152 	}
2153 	lock_rw_unlock(&az->lock);
2154 }
2155 
2156 /** find zones that are marked deleted and delete them.
2157  * This is called from apply_cfg, and there are no threads and no
2158  * workers, so the xfr can just be deleted. */
2159 static void
2160 az_delete_deleted_zones(struct auth_zones* az)
2161 {
2162 	struct auth_zone* z;
2163 	struct auth_zone* delete_list = NULL, *next;
2164 	struct auth_xfer* xfr;
2165 	lock_rw_wrlock(&az->lock);
2166 	RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2167 		lock_rw_wrlock(&z->lock);
2168 		if(z->zone_deleted) {
2169 			/* we cannot alter the rbtree right now, but
2170 			 * we can put it on a linked list and then
2171 			 * delete it */
2172 			z->delete_next = delete_list;
2173 			delete_list = z;
2174 		}
2175 		lock_rw_unlock(&z->lock);
2176 	}
2177 	/* now we are out of the tree loop and we can loop and delete
2178 	 * the zones */
2179 	z = delete_list;
2180 	while(z) {
2181 		next = z->delete_next;
2182 		xfr = auth_xfer_find(az, z->name, z->namelen, z->dclass);
2183 		if(xfr) {
2184 			(void)rbtree_delete(&az->xtree, &xfr->node);
2185 			auth_xfer_delete(xfr);
2186 		}
2187 		(void)rbtree_delete(&az->ztree, &z->node);
2188 		auth_zone_delete(z, az);
2189 		z = next;
2190 	}
2191 	lock_rw_unlock(&az->lock);
2192 }
2193 
2194 int auth_zones_apply_cfg(struct auth_zones* az, struct config_file* cfg,
2195 	int setup, int* is_rpz, struct module_env* env,
2196 	struct module_stack* mods)
2197 {
2198 	struct config_auth* p;
2199 	az_setall_deleted(az);
2200 	for(p = cfg->auths; p; p = p->next) {
2201 		if(!p->name || p->name[0] == 0) {
2202 			log_warn("auth-zone without a name, skipped");
2203 			continue;
2204 		}
2205 		*is_rpz = (*is_rpz || p->isrpz);
2206 		if(!auth_zones_cfg(az, p)) {
2207 			log_err("cannot config auth zone %s", p->name);
2208 			return 0;
2209 		}
2210 	}
2211 	az_delete_deleted_zones(az);
2212 	if(!auth_zones_read_zones(az, cfg, env, mods))
2213 		return 0;
2214 	if(setup) {
2215 		if(!auth_zones_setup_zones(az))
2216 			return 0;
2217 	}
2218 	return 1;
2219 }
2220 
2221 /** delete chunks
2222  * @param at: transfer structure with chunks list.  The chunks and their
2223  * 	data are freed.
2224  */
2225 static void
2226 auth_chunks_delete(struct auth_transfer* at)
2227 {
2228 	if(at->chunks_first) {
2229 		struct auth_chunk* c, *cn;
2230 		c = at->chunks_first;
2231 		while(c) {
2232 			cn = c->next;
2233 			free(c->data);
2234 			free(c);
2235 			c = cn;
2236 		}
2237 	}
2238 	at->chunks_first = NULL;
2239 	at->chunks_last = NULL;
2240 }
2241 
2242 /** free master addr list */
2243 static void
2244 auth_free_master_addrs(struct auth_addr* list)
2245 {
2246 	struct auth_addr *n;
2247 	while(list) {
2248 		n = list->next;
2249 		free(list);
2250 		list = n;
2251 	}
2252 }
2253 
2254 /** free the masters list */
2255 static void
2256 auth_free_masters(struct auth_master* list)
2257 {
2258 	struct auth_master* n;
2259 	while(list) {
2260 		n = list->next;
2261 		auth_free_master_addrs(list->list);
2262 		free(list->host);
2263 		free(list->file);
2264 		free(list);
2265 		list = n;
2266 	}
2267 }
2268 
2269 /** delete auth xfer structure
2270  * @param xfr: delete this xfer and its tasks.
2271  */
2272 static void
2273 auth_xfer_delete(struct auth_xfer* xfr)
2274 {
2275 	if(!xfr) return;
2276 	lock_basic_destroy(&xfr->lock);
2277 	free(xfr->name);
2278 	if(xfr->task_nextprobe) {
2279 		comm_timer_delete(xfr->task_nextprobe->timer);
2280 		free(xfr->task_nextprobe);
2281 	}
2282 	if(xfr->task_probe) {
2283 		auth_free_masters(xfr->task_probe->masters);
2284 		comm_point_delete(xfr->task_probe->cp);
2285 		comm_timer_delete(xfr->task_probe->timer);
2286 		free(xfr->task_probe);
2287 	}
2288 	if(xfr->task_transfer) {
2289 		auth_free_masters(xfr->task_transfer->masters);
2290 		comm_point_delete(xfr->task_transfer->cp);
2291 		comm_timer_delete(xfr->task_transfer->timer);
2292 		if(xfr->task_transfer->chunks_first) {
2293 			auth_chunks_delete(xfr->task_transfer);
2294 		}
2295 		free(xfr->task_transfer);
2296 	}
2297 	auth_free_masters(xfr->allow_notify_list);
2298 	free(xfr);
2299 }
2300 
2301 /** helper traverse to delete zones */
2302 static void
2303 auth_zone_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2304 {
2305 	struct auth_zone* z = (struct auth_zone*)n->key;
2306 	auth_zone_delete(z, NULL);
2307 }
2308 
2309 /** helper traverse to delete xfer zones */
2310 static void
2311 auth_xfer_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2312 {
2313 	struct auth_xfer* z = (struct auth_xfer*)n->key;
2314 	auth_xfer_delete(z);
2315 }
2316 
2317 void auth_zones_delete(struct auth_zones* az)
2318 {
2319 	if(!az) return;
2320 	lock_rw_destroy(&az->lock);
2321 	lock_rw_destroy(&az->rpz_lock);
2322 	traverse_postorder(&az->ztree, auth_zone_del, NULL);
2323 	traverse_postorder(&az->xtree, auth_xfer_del, NULL);
2324 	free(az);
2325 }
2326 
2327 /** true if domain has only nsec3 */
2328 static int
2329 domain_has_only_nsec3(struct auth_data* n)
2330 {
2331 	struct auth_rrset* rrset = n->rrsets;
2332 	int nsec3_seen = 0;
2333 	while(rrset) {
2334 		if(rrset->type == LDNS_RR_TYPE_NSEC3) {
2335 			nsec3_seen = 1;
2336 		} else if(rrset->type != LDNS_RR_TYPE_RRSIG) {
2337 			return 0;
2338 		}
2339 		rrset = rrset->next;
2340 	}
2341 	return nsec3_seen;
2342 }
2343 
2344 /** see if the domain has a wildcard child '*.domain' */
2345 static struct auth_data*
2346 az_find_wildcard_domain(struct auth_zone* z, uint8_t* nm, size_t nmlen)
2347 {
2348 	uint8_t wc[LDNS_MAX_DOMAINLEN];
2349 	if(nmlen+2 > sizeof(wc))
2350 		return NULL; /* result would be too long */
2351 	wc[0] = 1; /* length of wildcard label */
2352 	wc[1] = (uint8_t)'*'; /* wildcard label */
2353 	memmove(wc+2, nm, nmlen);
2354 	return az_find_name(z, wc, nmlen+2);
2355 }
2356 
2357 /** find wildcard between qname and cename */
2358 static struct auth_data*
2359 az_find_wildcard(struct auth_zone* z, struct query_info* qinfo,
2360 	struct auth_data* ce)
2361 {
2362 	uint8_t* nm = qinfo->qname;
2363 	size_t nmlen = qinfo->qname_len;
2364 	struct auth_data* node;
2365 	if(!dname_subdomain_c(nm, z->name))
2366 		return NULL; /* out of zone */
2367 	while((node=az_find_wildcard_domain(z, nm, nmlen))==NULL) {
2368 		/* see if we can go up to find the wildcard */
2369 		if(nmlen == z->namelen)
2370 			return NULL; /* top of zone reached */
2371 		if(ce && nmlen == ce->namelen)
2372 			return NULL; /* ce reached */
2373 		if(dname_is_root(nm))
2374 			return NULL; /* cannot go up */
2375 		dname_remove_label(&nm, &nmlen);
2376 	}
2377 	return node;
2378 }
2379 
2380 /** domain is not exact, find first candidate ce (name that matches
2381  * a part of qname) in tree */
2382 static struct auth_data*
2383 az_find_candidate_ce(struct auth_zone* z, struct query_info* qinfo,
2384 	struct auth_data* n)
2385 {
2386 	uint8_t* nm;
2387 	size_t nmlen;
2388 	if(n) {
2389 		nm = dname_get_shared_topdomain(qinfo->qname, n->name);
2390 	} else {
2391 		nm = qinfo->qname;
2392 	}
2393 	dname_count_size_labels(nm, &nmlen);
2394 	n = az_find_name(z, nm, nmlen);
2395 	/* delete labels and go up on name */
2396 	while(!n) {
2397 		if(dname_is_root(nm))
2398 			return NULL; /* cannot go up */
2399 		dname_remove_label(&nm, &nmlen);
2400 		n = az_find_name(z, nm, nmlen);
2401 	}
2402 	return n;
2403 }
2404 
2405 /** go up the auth tree to next existing name. */
2406 static struct auth_data*
2407 az_domain_go_up(struct auth_zone* z, struct auth_data* n)
2408 {
2409 	uint8_t* nm = n->name;
2410 	size_t nmlen = n->namelen;
2411 	while(!dname_is_root(nm)) {
2412 		dname_remove_label(&nm, &nmlen);
2413 		if((n=az_find_name(z, nm, nmlen)) != NULL)
2414 			return n;
2415 	}
2416 	return NULL;
2417 }
2418 
2419 /** Find the closest encloser, an name that exists and is above the
2420  * qname.
2421  * return true if the node (param node) is existing, nonobscured and
2422  * 	can be used to generate answers from.  It is then also node_exact.
2423  * returns false if the node is not good enough (or it wasn't node_exact)
2424  *	in this case the ce can be filled.
2425  *	if ce is NULL, no ce exists, and likely the zone is completely empty,
2426  *	not even with a zone apex.
2427  *	if ce is nonNULL it is the closest enclosing upper name (that exists
2428  *	itself for answer purposes).  That name may have DNAME, NS or wildcard
2429  *	rrset is the closest DNAME or NS rrset that was found.
2430  */
2431 static int
2432 az_find_ce(struct auth_zone* z, struct query_info* qinfo,
2433 	struct auth_data* node, int node_exact, struct auth_data** ce,
2434 	struct auth_rrset** rrset)
2435 {
2436 	struct auth_data* n = node;
2437 	*ce = NULL;
2438 	*rrset = NULL;
2439 	if(!node_exact) {
2440 		/* if not exact, lookup closest exact match */
2441 		n = az_find_candidate_ce(z, qinfo, n);
2442 	} else {
2443 		/* if exact, the node itself is the first candidate ce */
2444 		*ce = n;
2445 	}
2446 
2447 	/* no direct answer from nsec3-only domains */
2448 	if(n && domain_has_only_nsec3(n)) {
2449 		node_exact = 0;
2450 		*ce = NULL;
2451 	}
2452 
2453 	/* with exact matches, walk up the labels until we find the
2454 	 * delegation, or DNAME or zone end */
2455 	while(n) {
2456 		/* see if the current candidate has issues */
2457 		/* not zone apex and has type NS */
2458 		if(n->namelen != z->namelen &&
2459 			(*rrset=az_domain_rrset(n, LDNS_RR_TYPE_NS)) &&
2460 			/* delegate here, but DS at exact the dp has notype */
2461 			(qinfo->qtype != LDNS_RR_TYPE_DS ||
2462 			n->namelen != qinfo->qname_len)) {
2463 			/* referral */
2464 			/* this is ce and the lowernode is nonexisting */
2465 			*ce = n;
2466 			return 0;
2467 		}
2468 		/* not equal to qname and has type DNAME */
2469 		if(n->namelen != qinfo->qname_len &&
2470 			(*rrset=az_domain_rrset(n, LDNS_RR_TYPE_DNAME))) {
2471 			/* this is ce and the lowernode is nonexisting */
2472 			*ce = n;
2473 			return 0;
2474 		}
2475 
2476 		if(*ce == NULL && !domain_has_only_nsec3(n)) {
2477 			/* if not found yet, this exact name must be
2478 			 * our lowest match (but not nsec3onlydomain) */
2479 			*ce = n;
2480 		}
2481 
2482 		/* walk up the tree by removing labels from name and lookup */
2483 		n = az_domain_go_up(z, n);
2484 	}
2485 	/* found no problems, if it was an exact node, it is fine to use */
2486 	return node_exact;
2487 }
2488 
2489 /** add additional A/AAAA from domain names in rrset rdata (+offset)
2490  * offset is number of bytes in rdata where the dname is located. */
2491 static int
2492 az_add_additionals_from(struct auth_zone* z, struct regional* region,
2493 	struct dns_msg* msg, struct auth_rrset* rrset, size_t offset)
2494 {
2495 	struct packed_rrset_data* d = rrset->data;
2496 	size_t i;
2497 	if(!d) return 0;
2498 	for(i=0; i<d->count; i++) {
2499 		size_t dlen;
2500 		struct auth_data* domain;
2501 		struct auth_rrset* ref;
2502 		if(d->rr_len[i] < 2+offset)
2503 			continue; /* too short */
2504 		if(!(dlen = dname_valid(d->rr_data[i]+2+offset,
2505 			d->rr_len[i]-2-offset)))
2506 			continue; /* malformed */
2507 		domain = az_find_name(z, d->rr_data[i]+2+offset, dlen);
2508 		if(!domain)
2509 			continue;
2510 		if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_A)) != NULL) {
2511 			if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2512 				return 0;
2513 		}
2514 		if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_AAAA)) != NULL) {
2515 			if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2516 				return 0;
2517 		}
2518 	}
2519 	return 1;
2520 }
2521 
2522 /** add negative SOA record (with negative TTL) */
2523 static int
2524 az_add_negative_soa(struct auth_zone* z, struct regional* region,
2525 	struct dns_msg* msg)
2526 {
2527 	time_t minimum;
2528 	size_t i;
2529 	struct packed_rrset_data* d;
2530 	struct auth_rrset* soa;
2531 	struct auth_data* apex = az_find_name(z, z->name, z->namelen);
2532 	if(!apex) return 0;
2533 	soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
2534 	if(!soa) return 0;
2535 	/* must be first to put in message; we want to fix the TTL with
2536 	 * one RRset here, otherwise we'd need to loop over the RRs to get
2537 	 * the resulting lower TTL */
2538 	log_assert(msg->rep->rrset_count == 0);
2539 	if(!msg_add_rrset_ns(z, region, msg, apex, soa)) return 0;
2540 	/* fixup TTL */
2541 	d = (struct packed_rrset_data*)msg->rep->rrsets[msg->rep->rrset_count-1]->entry.data;
2542 	/* last 4 bytes are minimum ttl in network format */
2543 	if(d->count == 0) return 0;
2544 	if(d->rr_len[0] < 2+4) return 0;
2545 	minimum = (time_t)sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-4));
2546 	minimum = d->ttl<minimum?d->ttl:minimum;
2547 	d->ttl = minimum;
2548 	for(i=0; i < d->count + d->rrsig_count; i++)
2549 		d->rr_ttl[i] = minimum;
2550 	msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
2551 	msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
2552 	msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
2553 	return 1;
2554 }
2555 
2556 /** See if the query goes to empty nonterminal (that has no auth_data,
2557  * but there are nodes underneath.  We already checked that there are
2558  * not NS, or DNAME above, so that we only need to check if some node
2559  * exists below (with nonempty rr list), return true if emptynonterminal */
2560 static int
2561 az_empty_nonterminal(struct auth_zone* z, struct query_info* qinfo,
2562 	struct auth_data* node)
2563 {
2564 	struct auth_data* next;
2565 	if(!node) {
2566 		/* no smaller was found, use first (smallest) node as the
2567 		 * next one */
2568 		next = (struct auth_data*)rbtree_first(&z->data);
2569 	} else {
2570 		next = (struct auth_data*)rbtree_next(&node->node);
2571 	}
2572 	while(next && (rbnode_type*)next != RBTREE_NULL && next->rrsets == NULL) {
2573 		/* the next name has empty rrsets, is an empty nonterminal
2574 		 * itself, see if there exists something below it */
2575 		next = (struct auth_data*)rbtree_next(&node->node);
2576 	}
2577 	if((rbnode_type*)next == RBTREE_NULL || !next) {
2578 		/* there is no next node, so something below it cannot
2579 		 * exist */
2580 		return 0;
2581 	}
2582 	/* a next node exists, if there was something below the query,
2583 	 * this node has to be it.  See if it is below the query name */
2584 	if(dname_strict_subdomain_c(next->name, qinfo->qname))
2585 		return 1;
2586 	return 0;
2587 }
2588 
2589 /** create synth cname target name in buffer, or fail if too long */
2590 static size_t
2591 synth_cname_buf(uint8_t* qname, size_t qname_len, size_t dname_len,
2592 	uint8_t* dtarg, size_t dtarglen, uint8_t* buf, size_t buflen)
2593 {
2594 	size_t newlen = qname_len + dtarglen - dname_len;
2595 	if(newlen > buflen) {
2596 		/* YXDOMAIN error */
2597 		return 0;
2598 	}
2599 	/* new name is concatenation of qname front (without DNAME owner)
2600 	 * and DNAME target name */
2601 	memcpy(buf, qname, qname_len-dname_len);
2602 	memmove(buf+(qname_len-dname_len), dtarg, dtarglen);
2603 	return newlen;
2604 }
2605 
2606 /** create synthetic CNAME rrset for in a DNAME answer in region,
2607  * false on alloc failure, cname==NULL when name too long. */
2608 static int
2609 create_synth_cname(uint8_t* qname, size_t qname_len, struct regional* region,
2610 	struct auth_data* node, struct auth_rrset* dname, uint16_t dclass,
2611 	struct ub_packed_rrset_key** cname)
2612 {
2613 	uint8_t buf[LDNS_MAX_DOMAINLEN];
2614 	uint8_t* dtarg;
2615 	size_t dtarglen, newlen;
2616 	struct packed_rrset_data* d;
2617 
2618 	/* get DNAME target name */
2619 	if(dname->data->count < 1) return 0;
2620 	if(dname->data->rr_len[0] < 3) return 0; /* at least rdatalen +1 */
2621 	dtarg = dname->data->rr_data[0]+2;
2622 	dtarglen = dname->data->rr_len[0]-2;
2623 	if(sldns_read_uint16(dname->data->rr_data[0]) != dtarglen)
2624 		return 0; /* rdatalen in DNAME rdata is malformed */
2625 	if(dname_valid(dtarg, dtarglen) != dtarglen)
2626 		return 0; /* DNAME RR has malformed rdata */
2627 	if(qname_len == 0)
2628 		return 0; /* too short */
2629 	if(qname_len <= node->namelen)
2630 		return 0; /* qname too short for dname removal */
2631 
2632 	/* synthesize a CNAME */
2633 	newlen = synth_cname_buf(qname, qname_len, node->namelen,
2634 		dtarg, dtarglen, buf, sizeof(buf));
2635 	if(newlen == 0) {
2636 		/* YXDOMAIN error */
2637 		*cname = NULL;
2638 		return 1;
2639 	}
2640 	*cname = (struct ub_packed_rrset_key*)regional_alloc(region,
2641 		sizeof(struct ub_packed_rrset_key));
2642 	if(!*cname)
2643 		return 0; /* out of memory */
2644 	memset(&(*cname)->entry, 0, sizeof((*cname)->entry));
2645 	(*cname)->entry.key = (*cname);
2646 	(*cname)->rk.type = htons(LDNS_RR_TYPE_CNAME);
2647 	(*cname)->rk.rrset_class = htons(dclass);
2648 	(*cname)->rk.flags = 0;
2649 	(*cname)->rk.dname = regional_alloc_init(region, qname, qname_len);
2650 	if(!(*cname)->rk.dname)
2651 		return 0; /* out of memory */
2652 	(*cname)->rk.dname_len = qname_len;
2653 	(*cname)->entry.hash = rrset_key_hash(&(*cname)->rk);
2654 	d = (struct packed_rrset_data*)regional_alloc_zero(region,
2655 		sizeof(struct packed_rrset_data) + sizeof(size_t) +
2656 		sizeof(uint8_t*) + sizeof(time_t) + sizeof(uint16_t)
2657 		+ newlen);
2658 	if(!d)
2659 		return 0; /* out of memory */
2660 	(*cname)->entry.data = d;
2661 	d->ttl = 0; /* 0 for synthesized CNAME TTL */
2662 	d->count = 1;
2663 	d->rrsig_count = 0;
2664 	d->trust = rrset_trust_ans_noAA;
2665 	d->rr_len = (size_t*)((uint8_t*)d +
2666 		sizeof(struct packed_rrset_data));
2667 	d->rr_len[0] = newlen + sizeof(uint16_t);
2668 	packed_rrset_ptr_fixup(d);
2669 	d->rr_ttl[0] = d->ttl;
2670 	sldns_write_uint16(d->rr_data[0], newlen);
2671 	memmove(d->rr_data[0] + sizeof(uint16_t), buf, newlen);
2672 	return 1;
2673 }
2674 
2675 /** add a synthesized CNAME to the answer section */
2676 static int
2677 add_synth_cname(struct auth_zone* z, uint8_t* qname, size_t qname_len,
2678 	struct regional* region, struct dns_msg* msg, struct auth_data* dname,
2679 	struct auth_rrset* rrset)
2680 {
2681 	struct ub_packed_rrset_key* cname;
2682 	/* synthesize a CNAME */
2683 	if(!create_synth_cname(qname, qname_len, region, dname, rrset,
2684 		z->dclass, &cname)) {
2685 		/* out of memory */
2686 		return 0;
2687 	}
2688 	if(!cname) {
2689 		/* cname cannot be create because of YXDOMAIN */
2690 		msg->rep->flags |= LDNS_RCODE_YXDOMAIN;
2691 		return 1;
2692 	}
2693 	/* add cname to message */
2694 	if(!msg_grow_array(region, msg))
2695 		return 0;
2696 	msg->rep->rrsets[msg->rep->rrset_count] = cname;
2697 	msg->rep->rrset_count++;
2698 	msg->rep->an_numrrsets++;
2699 	msg_ttl(msg);
2700 	return 1;
2701 }
2702 
2703 /** Change a dname to a different one, for wildcard namechange */
2704 static void
2705 az_change_dnames(struct dns_msg* msg, uint8_t* oldname, uint8_t* newname,
2706 	size_t newlen, int an_only)
2707 {
2708 	size_t i;
2709 	size_t start = 0, end = msg->rep->rrset_count;
2710 	if(!an_only) start = msg->rep->an_numrrsets;
2711 	if(an_only) end = msg->rep->an_numrrsets;
2712 	for(i=start; i<end; i++) {
2713 		/* allocated in region so we can change the ptrs */
2714 		if(query_dname_compare(msg->rep->rrsets[i]->rk.dname, oldname)
2715 			== 0) {
2716 			msg->rep->rrsets[i]->rk.dname = newname;
2717 			msg->rep->rrsets[i]->rk.dname_len = newlen;
2718 		}
2719 	}
2720 }
2721 
2722 /** find NSEC record covering the query */
2723 static struct auth_rrset*
2724 az_find_nsec_cover(struct auth_zone* z, struct auth_data** node)
2725 {
2726 	uint8_t* nm = (*node)->name;
2727 	size_t nmlen = (*node)->namelen;
2728 	struct auth_rrset* rrset;
2729 	/* find the NSEC for the smallest-or-equal node */
2730 	/* if node == NULL, we did not find a smaller name.  But the zone
2731 	 * name is the smallest name and should have an NSEC. So there is
2732 	 * no NSEC to return (for a properly signed zone) */
2733 	/* for empty nonterminals, the auth-data node should not exist,
2734 	 * and thus we don't need to go rbtree_previous here to find
2735 	 * a domain with an NSEC record */
2736 	/* but there could be glue, and if this is node, then it has no NSEC.
2737 	 * Go up to find nonglue (previous) NSEC-holding nodes */
2738 	while((rrset=az_domain_rrset(*node, LDNS_RR_TYPE_NSEC)) == NULL) {
2739 		if(dname_is_root(nm)) return NULL;
2740 		if(nmlen == z->namelen) return NULL;
2741 		dname_remove_label(&nm, &nmlen);
2742 		/* adjust *node for the nsec rrset to find in */
2743 		*node = az_find_name(z, nm, nmlen);
2744 	}
2745 	return rrset;
2746 }
2747 
2748 /** Find NSEC and add for wildcard denial */
2749 static int
2750 az_nsec_wildcard_denial(struct auth_zone* z, struct regional* region,
2751 	struct dns_msg* msg, uint8_t* cenm, size_t cenmlen)
2752 {
2753 	struct query_info qinfo;
2754 	int node_exact;
2755 	struct auth_data* node;
2756 	struct auth_rrset* nsec;
2757 	uint8_t wc[LDNS_MAX_DOMAINLEN];
2758 	if(cenmlen+2 > sizeof(wc))
2759 		return 0; /* result would be too long */
2760 	wc[0] = 1; /* length of wildcard label */
2761 	wc[1] = (uint8_t)'*'; /* wildcard label */
2762 	memmove(wc+2, cenm, cenmlen);
2763 
2764 	/* we have '*.ce' in wc wildcard name buffer */
2765 	/* get nsec cover for that */
2766 	qinfo.qname = wc;
2767 	qinfo.qname_len = cenmlen+2;
2768 	qinfo.qtype = 0;
2769 	qinfo.qclass = 0;
2770 	az_find_domain(z, &qinfo, &node_exact, &node);
2771 	if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
2772 		if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
2773 	}
2774 	return 1;
2775 }
2776 
2777 /** Find the NSEC3PARAM rrset (if any) and if true you have the parameters */
2778 static int
2779 az_nsec3_param(struct auth_zone* z, int* algo, size_t* iter, uint8_t** salt,
2780 	size_t* saltlen)
2781 {
2782 	struct auth_data* apex;
2783 	struct auth_rrset* param;
2784 	size_t i;
2785 	apex = az_find_name(z, z->name, z->namelen);
2786 	if(!apex) return 0;
2787 	param = az_domain_rrset(apex, LDNS_RR_TYPE_NSEC3PARAM);
2788 	if(!param || param->data->count==0)
2789 		return 0; /* no RRset or no RRs in rrset */
2790 	/* find out which NSEC3PARAM RR has supported parameters */
2791 	/* skip unknown flags (dynamic signer is recalculating nsec3 chain) */
2792 	for(i=0; i<param->data->count; i++) {
2793 		uint8_t* rdata = param->data->rr_data[i]+2;
2794 		size_t rdatalen = param->data->rr_len[i];
2795 		if(rdatalen < 2+5)
2796 			continue; /* too short */
2797 		if(!nsec3_hash_algo_size_supported((int)(rdata[0])))
2798 			continue; /* unsupported algo */
2799 		if(rdatalen < (size_t)(2+5+(size_t)rdata[4]))
2800 			continue; /* salt missing */
2801 		if((rdata[1]&NSEC3_UNKNOWN_FLAGS)!=0)
2802 			continue; /* unknown flags */
2803 		*algo = (int)(rdata[0]);
2804 		*iter = sldns_read_uint16(rdata+2);
2805 		*saltlen = rdata[4];
2806 		if(*saltlen == 0)
2807 			*salt = NULL;
2808 		else	*salt = rdata+5;
2809 		return 1;
2810 	}
2811 	/* no supported params */
2812 	return 0;
2813 }
2814 
2815 /** Hash a name with nsec3param into buffer, it has zone name appended.
2816  * return length of hash */
2817 static size_t
2818 az_nsec3_hash(uint8_t* buf, size_t buflen, uint8_t* nm, size_t nmlen,
2819 	int algo, size_t iter, uint8_t* salt, size_t saltlen)
2820 {
2821 	size_t hlen = nsec3_hash_algo_size_supported(algo);
2822 	/* buffer has domain name, nsec3hash, and 256 is for max saltlen
2823 	 * (salt has 0-255 length) */
2824 	unsigned char p[LDNS_MAX_DOMAINLEN+1+N3HASHBUFLEN+256];
2825 	size_t i;
2826 	if(nmlen+saltlen > sizeof(p) || hlen+saltlen > sizeof(p))
2827 		return 0;
2828 	if(hlen > buflen)
2829 		return 0; /* somehow too large for destination buffer */
2830 	/* hashfunc(name, salt) */
2831 	memmove(p, nm, nmlen);
2832 	query_dname_tolower(p);
2833 	if(salt && saltlen > 0)
2834 		memmove(p+nmlen, salt, saltlen);
2835 	(void)secalgo_nsec3_hash(algo, p, nmlen+saltlen, (unsigned char*)buf);
2836 	for(i=0; i<iter; i++) {
2837 		/* hashfunc(hash, salt) */
2838 		memmove(p, buf, hlen);
2839 		if(salt && saltlen > 0)
2840 			memmove(p+hlen, salt, saltlen);
2841 		(void)secalgo_nsec3_hash(algo, p, hlen+saltlen,
2842 			(unsigned char*)buf);
2843 	}
2844 	return hlen;
2845 }
2846 
2847 /** Hash name and return b32encoded hashname for lookup, zone name appended */
2848 static int
2849 az_nsec3_hashname(struct auth_zone* z, uint8_t* hashname, size_t* hashnmlen,
2850 	uint8_t* nm, size_t nmlen, int algo, size_t iter, uint8_t* salt,
2851 	size_t saltlen)
2852 {
2853 	uint8_t hash[N3HASHBUFLEN];
2854 	size_t hlen;
2855 	int ret;
2856 	hlen = az_nsec3_hash(hash, sizeof(hash), nm, nmlen, algo, iter,
2857 		salt, saltlen);
2858 	if(!hlen) return 0;
2859 	/* b32 encode */
2860 	if(*hashnmlen < hlen*2+1+z->namelen) /* approx b32 as hexb16 */
2861 		return 0;
2862 	ret = sldns_b32_ntop_extended_hex(hash, hlen, (char*)(hashname+1),
2863 		(*hashnmlen)-1);
2864 	if(ret<1)
2865 		return 0;
2866 	hashname[0] = (uint8_t)ret;
2867 	ret++;
2868 	if((*hashnmlen) - ret < z->namelen)
2869 		return 0;
2870 	memmove(hashname+ret, z->name, z->namelen);
2871 	*hashnmlen = z->namelen+(size_t)ret;
2872 	return 1;
2873 }
2874 
2875 /** Find the datanode that covers the nsec3hash-name */
2876 static struct auth_data*
2877 az_nsec3_findnode(struct auth_zone* z, uint8_t* hashnm, size_t hashnmlen)
2878 {
2879 	struct query_info qinfo;
2880 	struct auth_data* node;
2881 	int node_exact;
2882 	qinfo.qclass = 0;
2883 	qinfo.qtype = 0;
2884 	qinfo.qname = hashnm;
2885 	qinfo.qname_len = hashnmlen;
2886 	/* because canonical ordering and b32 nsec3 ordering are the same.
2887 	 * this is a good lookup to find the nsec3 name. */
2888 	az_find_domain(z, &qinfo, &node_exact, &node);
2889 	/* but we may have to skip non-nsec3 nodes */
2890 	/* this may be a lot, the way to speed that up is to have a
2891 	 * separate nsec3 tree with nsec3 nodes */
2892 	while(node && (rbnode_type*)node != RBTREE_NULL &&
2893 		!az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2894 		node = (struct auth_data*)rbtree_previous(&node->node);
2895 	}
2896 	if((rbnode_type*)node == RBTREE_NULL)
2897 		node = NULL;
2898 	return node;
2899 }
2900 
2901 /** Find cover for hashed(nm, nmlen) (or NULL) */
2902 static struct auth_data*
2903 az_nsec3_find_cover(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2904 	int algo, size_t iter, uint8_t* salt, size_t saltlen)
2905 {
2906 	struct auth_data* node;
2907 	uint8_t hname[LDNS_MAX_DOMAINLEN];
2908 	size_t hlen = sizeof(hname);
2909 	if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2910 		salt, saltlen))
2911 		return NULL;
2912 	node = az_nsec3_findnode(z, hname, hlen);
2913 	if(node)
2914 		return node;
2915 	/* we did not find any, perhaps because the NSEC3 hash is before
2916 	 * the first hash, we have to find the 'last hash' in the zone */
2917 	node = (struct auth_data*)rbtree_last(&z->data);
2918 	while(node && (rbnode_type*)node != RBTREE_NULL &&
2919 		!az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2920 		node = (struct auth_data*)rbtree_previous(&node->node);
2921 	}
2922 	if((rbnode_type*)node == RBTREE_NULL)
2923 		node = NULL;
2924 	return node;
2925 }
2926 
2927 /** Find exact match for hashed(nm, nmlen) NSEC3 record or NULL */
2928 static struct auth_data*
2929 az_nsec3_find_exact(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2930 	int algo, size_t iter, uint8_t* salt, size_t saltlen)
2931 {
2932 	struct auth_data* node;
2933 	uint8_t hname[LDNS_MAX_DOMAINLEN];
2934 	size_t hlen = sizeof(hname);
2935 	if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2936 		salt, saltlen))
2937 		return NULL;
2938 	node = az_find_name(z, hname, hlen);
2939 	if(az_domain_rrset(node, LDNS_RR_TYPE_NSEC3))
2940 		return node;
2941 	return NULL;
2942 }
2943 
2944 /** Return nextcloser name (as a ref into the qname).  This is one label
2945  * more than the cenm (cename must be a suffix of qname) */
2946 static void
2947 az_nsec3_get_nextcloser(uint8_t* cenm, uint8_t* qname, size_t qname_len,
2948 	uint8_t** nx, size_t* nxlen)
2949 {
2950 	int celabs = dname_count_labels(cenm);
2951 	int qlabs = dname_count_labels(qname);
2952 	int strip = qlabs - celabs -1;
2953 	log_assert(dname_strict_subdomain(qname, qlabs, cenm, celabs));
2954 	*nx = qname;
2955 	*nxlen = qname_len;
2956 	if(strip>0)
2957 		dname_remove_labels(nx, nxlen, strip);
2958 }
2959 
2960 /** Find the closest encloser that has exact NSEC3.
2961  * updated cenm to the new name. If it went up no-exact-ce is true. */
2962 static struct auth_data*
2963 az_nsec3_find_ce(struct auth_zone* z, uint8_t** cenm, size_t* cenmlen,
2964 	int* no_exact_ce, int algo, size_t iter, uint8_t* salt, size_t saltlen)
2965 {
2966 	struct auth_data* node;
2967 	while((node = az_nsec3_find_exact(z, *cenm, *cenmlen,
2968 		algo, iter, salt, saltlen)) == NULL) {
2969 		if(*cenmlen == z->namelen) {
2970 			/* next step up would take us out of the zone. fail */
2971 			return NULL;
2972 		}
2973 		*no_exact_ce = 1;
2974 		dname_remove_label(cenm, cenmlen);
2975 	}
2976 	return node;
2977 }
2978 
2979 /* Insert NSEC3 record in authority section, if NULL does nothing */
2980 static int
2981 az_nsec3_insert(struct auth_zone* z, struct regional* region,
2982 	struct dns_msg* msg, struct auth_data* node)
2983 {
2984 	struct auth_rrset* nsec3;
2985 	if(!node) return 1; /* no node, skip this */
2986 	nsec3 = az_domain_rrset(node, LDNS_RR_TYPE_NSEC3);
2987 	if(!nsec3) return 1; /* if no nsec3 RR, skip it */
2988 	if(!msg_add_rrset_ns(z, region, msg, node, nsec3)) return 0;
2989 	return 1;
2990 }
2991 
2992 /** add NSEC3 records to the zone for the nsec3 proof.
2993  * Specify with the flags with parts of the proof are required.
2994  * the ce is the exact matching name (for notype) but also delegation points.
2995  * qname is the one where the nextcloser name can be derived from.
2996  * If NSEC3 is not properly there (in the zone) nothing is added.
2997  * always enabled: include nsec3 proving about the Closest Encloser.
2998  * 	that is an exact match that should exist for it.
2999  * 	If that does not exist, a higher exact match + nxproof is enabled
3000  * 	(for some sort of opt-out empty nonterminal cases).
3001  * nodataproof: search for exact match and include that instead.
3002  * ceproof: include ce proof NSEC3 (omitted for wildcard replies).
3003  * nxproof: include denial of the qname.
3004  * wcproof: include denial of wildcard (wildcard.ce).
3005  */
3006 static int
3007 az_add_nsec3_proof(struct auth_zone* z, struct regional* region,
3008 	struct dns_msg* msg, uint8_t* cenm, size_t cenmlen, uint8_t* qname,
3009 	size_t qname_len, int nodataproof, int ceproof, int nxproof,
3010 	int wcproof)
3011 {
3012 	int algo;
3013 	size_t iter, saltlen;
3014 	uint8_t* salt;
3015 	int no_exact_ce = 0;
3016 	struct auth_data* node;
3017 
3018 	/* find parameters of nsec3 proof */
3019 	if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen))
3020 		return 1; /* no nsec3 */
3021 	if(nodataproof) {
3022 		/* see if the node has a hash of itself for the nodata
3023 		 * proof nsec3, this has to be an exact match nsec3. */
3024 		struct auth_data* match;
3025 		match = az_nsec3_find_exact(z, qname, qname_len, algo,
3026 			iter, salt, saltlen);
3027 		if(match) {
3028 			if(!az_nsec3_insert(z, region, msg, match))
3029 				return 0;
3030 			/* only nodata NSEC3 needed, no CE or others. */
3031 			return 1;
3032 		}
3033 	}
3034 	/* find ce that has an NSEC3 */
3035 	if(ceproof) {
3036 		node = az_nsec3_find_ce(z, &cenm, &cenmlen, &no_exact_ce,
3037 			algo, iter, salt, saltlen);
3038 		if(no_exact_ce) nxproof = 1;
3039 		if(!az_nsec3_insert(z, region, msg, node))
3040 			return 0;
3041 	}
3042 
3043 	if(nxproof) {
3044 		uint8_t* nx;
3045 		size_t nxlen;
3046 		/* create nextcloser domain name */
3047 		az_nsec3_get_nextcloser(cenm, qname, qname_len, &nx, &nxlen);
3048 		/* find nsec3 that matches or covers it */
3049 		node = az_nsec3_find_cover(z, nx, nxlen, algo, iter, salt,
3050 			saltlen);
3051 		if(!az_nsec3_insert(z, region, msg, node))
3052 			return 0;
3053 	}
3054 	if(wcproof) {
3055 		/* create wildcard name *.ce */
3056 		uint8_t wc[LDNS_MAX_DOMAINLEN];
3057 		size_t wclen;
3058 		if(cenmlen+2 > sizeof(wc))
3059 			return 0; /* result would be too long */
3060 		wc[0] = 1; /* length of wildcard label */
3061 		wc[1] = (uint8_t)'*'; /* wildcard label */
3062 		memmove(wc+2, cenm, cenmlen);
3063 		wclen = cenmlen+2;
3064 		/* find nsec3 that matches or covers it */
3065 		node = az_nsec3_find_cover(z, wc, wclen, algo, iter, salt,
3066 			saltlen);
3067 		if(!az_nsec3_insert(z, region, msg, node))
3068 			return 0;
3069 	}
3070 	return 1;
3071 }
3072 
3073 /** generate answer for positive answer */
3074 static int
3075 az_generate_positive_answer(struct auth_zone* z, struct regional* region,
3076 	struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
3077 {
3078 	if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3079 	/* see if we want additional rrs */
3080 	if(rrset->type == LDNS_RR_TYPE_MX) {
3081 		if(!az_add_additionals_from(z, region, msg, rrset, 2))
3082 			return 0;
3083 	} else if(rrset->type == LDNS_RR_TYPE_SRV) {
3084 		if(!az_add_additionals_from(z, region, msg, rrset, 6))
3085 			return 0;
3086 	} else if(rrset->type == LDNS_RR_TYPE_NS) {
3087 		if(!az_add_additionals_from(z, region, msg, rrset, 0))
3088 			return 0;
3089 	}
3090 	return 1;
3091 }
3092 
3093 /** generate answer for type ANY answer */
3094 static int
3095 az_generate_any_answer(struct auth_zone* z, struct regional* region,
3096 	struct dns_msg* msg, struct auth_data* node)
3097 {
3098 	struct auth_rrset* rrset;
3099 	int added = 0;
3100 	/* add a couple (at least one) RRs */
3101 	if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_SOA)) != NULL) {
3102 		if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3103 		added++;
3104 	}
3105 	if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_MX)) != NULL) {
3106 		if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3107 		added++;
3108 	}
3109 	if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_A)) != NULL) {
3110 		if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3111 		added++;
3112 	}
3113 	if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_AAAA)) != NULL) {
3114 		if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3115 		added++;
3116 	}
3117 	if(added == 0 && node && node->rrsets) {
3118 		if(!msg_add_rrset_an(z, region, msg, node,
3119 			node->rrsets)) return 0;
3120 	}
3121 	return 1;
3122 }
3123 
3124 /** follow cname chain and add more data to the answer section */
3125 static int
3126 follow_cname_chain(struct auth_zone* z, uint16_t qtype,
3127 	struct regional* region, struct dns_msg* msg,
3128 	struct packed_rrset_data* d)
3129 {
3130 	int maxchain = 0;
3131 	/* see if we can add the target of the CNAME into the answer */
3132 	while(maxchain++ < MAX_CNAME_CHAIN) {
3133 		struct auth_data* node;
3134 		struct auth_rrset* rrset;
3135 		size_t clen;
3136 		/* d has cname rdata */
3137 		if(d->count == 0) break; /* no CNAME */
3138 		if(d->rr_len[0] < 2+1) break; /* too small */
3139 		if((clen=dname_valid(d->rr_data[0]+2, d->rr_len[0]-2))==0)
3140 			break; /* malformed */
3141 		if(!dname_subdomain_c(d->rr_data[0]+2, z->name))
3142 			break; /* target out of zone */
3143 		if((node = az_find_name(z, d->rr_data[0]+2, clen))==NULL)
3144 			break; /* no such target name */
3145 		if((rrset=az_domain_rrset(node, qtype))!=NULL) {
3146 			/* done we found the target */
3147 			if(!msg_add_rrset_an(z, region, msg, node, rrset))
3148 				return 0;
3149 			break;
3150 		}
3151 		if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME))==NULL)
3152 			break; /* no further CNAME chain, notype */
3153 		if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3154 		d = rrset->data;
3155 	}
3156 	return 1;
3157 }
3158 
3159 /** generate answer for cname answer */
3160 static int
3161 az_generate_cname_answer(struct auth_zone* z, struct query_info* qinfo,
3162 	struct regional* region, struct dns_msg* msg,
3163 	struct auth_data* node, struct auth_rrset* rrset)
3164 {
3165 	if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3166 	if(!rrset) return 1;
3167 	if(!follow_cname_chain(z, qinfo->qtype, region, msg, rrset->data))
3168 		return 0;
3169 	return 1;
3170 }
3171 
3172 /** generate answer for notype answer */
3173 static int
3174 az_generate_notype_answer(struct auth_zone* z, struct regional* region,
3175 	struct dns_msg* msg, struct auth_data* node)
3176 {
3177 	struct auth_rrset* rrset;
3178 	if(!az_add_negative_soa(z, region, msg)) return 0;
3179 	/* DNSSEC denial NSEC */
3180 	if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_NSEC))!=NULL) {
3181 		if(!msg_add_rrset_ns(z, region, msg, node, rrset)) return 0;
3182 	} else if(node) {
3183 		/* DNSSEC denial NSEC3 */
3184 		if(!az_add_nsec3_proof(z, region, msg, node->name,
3185 			node->namelen, msg->qinfo.qname,
3186 			msg->qinfo.qname_len, 1, 1, 0, 0))
3187 			return 0;
3188 	}
3189 	return 1;
3190 }
3191 
3192 /** generate answer for referral answer */
3193 static int
3194 az_generate_referral_answer(struct auth_zone* z, struct regional* region,
3195 	struct dns_msg* msg, struct auth_data* ce, struct auth_rrset* rrset)
3196 {
3197 	struct auth_rrset* ds, *nsec;
3198 	/* turn off AA flag, referral is nonAA because it leaves the zone */
3199 	log_assert(ce);
3200 	msg->rep->flags &= ~BIT_AA;
3201 	if(!msg_add_rrset_ns(z, region, msg, ce, rrset)) return 0;
3202 	/* add DS or deny it */
3203 	if((ds=az_domain_rrset(ce, LDNS_RR_TYPE_DS))!=NULL) {
3204 		if(!msg_add_rrset_ns(z, region, msg, ce, ds)) return 0;
3205 	} else {
3206 		/* deny the DS */
3207 		if((nsec=az_domain_rrset(ce, LDNS_RR_TYPE_NSEC))!=NULL) {
3208 			if(!msg_add_rrset_ns(z, region, msg, ce, nsec))
3209 				return 0;
3210 		} else {
3211 			if(!az_add_nsec3_proof(z, region, msg, ce->name,
3212 				ce->namelen, msg->qinfo.qname,
3213 				msg->qinfo.qname_len, 1, 1, 0, 0))
3214 				return 0;
3215 		}
3216 	}
3217 	/* add additional rrs for type NS */
3218 	if(!az_add_additionals_from(z, region, msg, rrset, 0)) return 0;
3219 	return 1;
3220 }
3221 
3222 /** generate answer for DNAME answer */
3223 static int
3224 az_generate_dname_answer(struct auth_zone* z, struct query_info* qinfo,
3225 	struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3226 	struct auth_rrset* rrset)
3227 {
3228 	log_assert(ce);
3229 	/* add the DNAME and then a CNAME */
3230 	if(!msg_add_rrset_an(z, region, msg, ce, rrset)) return 0;
3231 	if(!add_synth_cname(z, qinfo->qname, qinfo->qname_len, region,
3232 		msg, ce, rrset)) return 0;
3233 	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_YXDOMAIN)
3234 		return 1;
3235 	if(msg->rep->rrset_count == 0 ||
3236 		!msg->rep->rrsets[msg->rep->rrset_count-1])
3237 		return 0;
3238 	if(!follow_cname_chain(z, qinfo->qtype, region, msg,
3239 		(struct packed_rrset_data*)msg->rep->rrsets[
3240 		msg->rep->rrset_count-1]->entry.data))
3241 		return 0;
3242 	return 1;
3243 }
3244 
3245 /** generate answer for wildcard answer */
3246 static int
3247 az_generate_wildcard_answer(struct auth_zone* z, struct query_info* qinfo,
3248 	struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3249 	struct auth_data* wildcard, struct auth_data* node)
3250 {
3251 	struct auth_rrset* rrset, *nsec;
3252 	int insert_ce = 0;
3253 	if((rrset=az_domain_rrset(wildcard, qinfo->qtype)) != NULL) {
3254 		/* wildcard has type, add it */
3255 		if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
3256 			return 0;
3257 		az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3258 			msg->qinfo.qname_len, 1);
3259 	} else if((rrset=az_domain_rrset(wildcard, LDNS_RR_TYPE_CNAME))!=NULL) {
3260 		/* wildcard has cname instead, do that */
3261 		if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
3262 			return 0;
3263 		az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3264 			msg->qinfo.qname_len, 1);
3265 		if(!follow_cname_chain(z, qinfo->qtype, region, msg,
3266 			rrset->data))
3267 			return 0;
3268 	} else if(qinfo->qtype == LDNS_RR_TYPE_ANY && wildcard->rrsets) {
3269 		/* add ANY rrsets from wildcard node */
3270 		if(!az_generate_any_answer(z, region, msg, wildcard))
3271 			return 0;
3272 		az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3273 			msg->qinfo.qname_len, 1);
3274 	} else {
3275 		/* wildcard has nodata, notype answer */
3276 		/* call other notype routine for dnssec notype denials */
3277 		if(!az_generate_notype_answer(z, region, msg, wildcard))
3278 			return 0;
3279 		/* because the notype, there is no positive data with an
3280 		 * RRSIG that indicates the wildcard position.  Thus the
3281 		 * wildcard qname denial needs to have a CE nsec3. */
3282 		insert_ce = 1;
3283 	}
3284 
3285 	/* ce and node for dnssec denial of wildcard original name */
3286 	if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
3287 		if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
3288 	} else if(ce) {
3289 		uint8_t* wildup = wildcard->name;
3290 		size_t wilduplen= wildcard->namelen;
3291 		dname_remove_label(&wildup, &wilduplen);
3292 		if(!az_add_nsec3_proof(z, region, msg, wildup,
3293 			wilduplen, msg->qinfo.qname,
3294 			msg->qinfo.qname_len, 0, insert_ce, 1, 0))
3295 			return 0;
3296 	}
3297 
3298 	/* fixup name of wildcard from *.zone to qname, use already allocated
3299 	 * pointer to msg qname */
3300 	az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3301 		msg->qinfo.qname_len, 0);
3302 	return 1;
3303 }
3304 
3305 /** generate answer for nxdomain answer */
3306 static int
3307 az_generate_nxdomain_answer(struct auth_zone* z, struct regional* region,
3308 	struct dns_msg* msg, struct auth_data* ce, struct auth_data* node)
3309 {
3310 	struct auth_rrset* nsec;
3311 	msg->rep->flags |= LDNS_RCODE_NXDOMAIN;
3312 	if(!az_add_negative_soa(z, region, msg)) return 0;
3313 	if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
3314 		if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
3315 		if(ce && !az_nsec_wildcard_denial(z, region, msg, ce->name,
3316 			ce->namelen)) return 0;
3317 	} else if(ce) {
3318 		if(!az_add_nsec3_proof(z, region, msg, ce->name,
3319 			ce->namelen, msg->qinfo.qname,
3320 			msg->qinfo.qname_len, 0, 1, 1, 1))
3321 			return 0;
3322 	}
3323 	return 1;
3324 }
3325 
3326 /** Create answers when an exact match exists for the domain name */
3327 static int
3328 az_generate_answer_with_node(struct auth_zone* z, struct query_info* qinfo,
3329 	struct regional* region, struct dns_msg* msg, struct auth_data* node)
3330 {
3331 	struct auth_rrset* rrset;
3332 	/* positive answer, rrset we are looking for exists */
3333 	if((rrset=az_domain_rrset(node, qinfo->qtype)) != NULL) {
3334 		return az_generate_positive_answer(z, region, msg, node, rrset);
3335 	}
3336 	/* CNAME? */
3337 	if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME)) != NULL) {
3338 		return az_generate_cname_answer(z, qinfo, region, msg,
3339 			node, rrset);
3340 	}
3341 	/* type ANY ? */
3342 	if(qinfo->qtype == LDNS_RR_TYPE_ANY) {
3343 		return az_generate_any_answer(z, region, msg, node);
3344 	}
3345 	/* NOERROR/NODATA (no such type at domain name) */
3346 	return az_generate_notype_answer(z, region, msg, node);
3347 }
3348 
3349 /** Generate answer without an existing-node that we can use.
3350  * So it'll be a referral, DNAME or nxdomain */
3351 static int
3352 az_generate_answer_nonexistnode(struct auth_zone* z, struct query_info* qinfo,
3353 	struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3354 	struct auth_rrset* rrset, struct auth_data* node)
3355 {
3356 	struct auth_data* wildcard;
3357 
3358 	/* we do not have an exact matching name (that exists) */
3359 	/* see if we have a NS or DNAME in the ce */
3360 	if(ce && rrset && rrset->type == LDNS_RR_TYPE_NS) {
3361 		return az_generate_referral_answer(z, region, msg, ce, rrset);
3362 	}
3363 	if(ce && rrset && rrset->type == LDNS_RR_TYPE_DNAME) {
3364 		return az_generate_dname_answer(z, qinfo, region, msg, ce,
3365 			rrset);
3366 	}
3367 	/* if there is an empty nonterminal, wildcard and nxdomain don't
3368 	 * happen, it is a notype answer */
3369 	if(az_empty_nonterminal(z, qinfo, node)) {
3370 		return az_generate_notype_answer(z, region, msg, node);
3371 	}
3372 	/* see if we have a wildcard under the ce */
3373 	if((wildcard=az_find_wildcard(z, qinfo, ce)) != NULL) {
3374 		return az_generate_wildcard_answer(z, qinfo, region, msg,
3375 			ce, wildcard, node);
3376 	}
3377 	/* generate nxdomain answer */
3378 	return az_generate_nxdomain_answer(z, region, msg, ce, node);
3379 }
3380 
3381 /** Lookup answer in a zone. */
3382 static int
3383 auth_zone_generate_answer(struct auth_zone* z, struct query_info* qinfo,
3384 	struct regional* region, struct dns_msg** msg, int* fallback)
3385 {
3386 	struct auth_data* node, *ce;
3387 	struct auth_rrset* rrset;
3388 	int node_exact, node_exists;
3389 	/* does the zone want fallback in case of failure? */
3390 	*fallback = z->fallback_enabled;
3391 	if(!(*msg=msg_create(region, qinfo))) return 0;
3392 
3393 	/* lookup if there is a matching domain name for the query */
3394 	az_find_domain(z, qinfo, &node_exact, &node);
3395 
3396 	/* see if node exists for generating answers from (i.e. not glue and
3397 	 * obscured by NS or DNAME or NSEC3-only), and also return the
3398 	 * closest-encloser from that, closest node that should be used
3399 	 * to generate answers from that is above the query */
3400 	node_exists = az_find_ce(z, qinfo, node, node_exact, &ce, &rrset);
3401 
3402 	if(verbosity >= VERB_ALGO) {
3403 		char zname[256], qname[256], nname[256], cename[256],
3404 			tpstr[32], rrstr[32];
3405 		sldns_wire2str_dname_buf(qinfo->qname, qinfo->qname_len, qname,
3406 			sizeof(qname));
3407 		sldns_wire2str_type_buf(qinfo->qtype, tpstr, sizeof(tpstr));
3408 		sldns_wire2str_dname_buf(z->name, z->namelen, zname,
3409 			sizeof(zname));
3410 		if(node)
3411 			sldns_wire2str_dname_buf(node->name, node->namelen,
3412 				nname, sizeof(nname));
3413 		else	snprintf(nname, sizeof(nname), "NULL");
3414 		if(ce)
3415 			sldns_wire2str_dname_buf(ce->name, ce->namelen,
3416 				cename, sizeof(cename));
3417 		else	snprintf(cename, sizeof(cename), "NULL");
3418 		if(rrset) sldns_wire2str_type_buf(rrset->type, rrstr,
3419 			sizeof(rrstr));
3420 		else	snprintf(rrstr, sizeof(rrstr), "NULL");
3421 		log_info("auth_zone %s query %s %s, domain %s %s %s, "
3422 			"ce %s, rrset %s", zname, qname, tpstr, nname,
3423 			(node_exact?"exact":"notexact"),
3424 			(node_exists?"exist":"notexist"), cename, rrstr);
3425 	}
3426 
3427 	if(node_exists) {
3428 		/* the node is fine, generate answer from node */
3429 		return az_generate_answer_with_node(z, qinfo, region, *msg,
3430 			node);
3431 	}
3432 	return az_generate_answer_nonexistnode(z, qinfo, region, *msg,
3433 		ce, rrset, node);
3434 }
3435 
3436 int auth_zones_lookup(struct auth_zones* az, struct query_info* qinfo,
3437 	struct regional* region, struct dns_msg** msg, int* fallback,
3438 	uint8_t* dp_nm, size_t dp_nmlen)
3439 {
3440 	int r;
3441 	struct auth_zone* z;
3442 	/* find the zone that should contain the answer. */
3443 	lock_rw_rdlock(&az->lock);
3444 	z = auth_zone_find(az, dp_nm, dp_nmlen, qinfo->qclass);
3445 	if(!z) {
3446 		lock_rw_unlock(&az->lock);
3447 		/* no auth zone, fallback to internet */
3448 		*fallback = 1;
3449 		return 0;
3450 	}
3451 	lock_rw_rdlock(&z->lock);
3452 	lock_rw_unlock(&az->lock);
3453 
3454 	/* if not for upstream queries, fallback */
3455 	if(!z->for_upstream) {
3456 		lock_rw_unlock(&z->lock);
3457 		*fallback = 1;
3458 		return 0;
3459 	}
3460 	if(z->zone_expired) {
3461 		*fallback = z->fallback_enabled;
3462 		lock_rw_unlock(&z->lock);
3463 		return 0;
3464 	}
3465 	/* see what answer that zone would generate */
3466 	r = auth_zone_generate_answer(z, qinfo, region, msg, fallback);
3467 	lock_rw_unlock(&z->lock);
3468 	return r;
3469 }
3470 
3471 /** encode auth answer */
3472 static void
3473 auth_answer_encode(struct query_info* qinfo, struct module_env* env,
3474 	struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3475 	struct regional* temp, struct dns_msg* msg)
3476 {
3477 	uint16_t udpsize;
3478 	udpsize = edns->udp_size;
3479 	edns->edns_version = EDNS_ADVERTISED_VERSION;
3480 	edns->udp_size = EDNS_ADVERTISED_SIZE;
3481 	edns->ext_rcode = 0;
3482 	edns->bits &= EDNS_DO;
3483 
3484 	if(!inplace_cb_reply_local_call(env, qinfo, NULL, msg->rep,
3485 		(int)FLAGS_GET_RCODE(msg->rep->flags), edns, repinfo, temp, env->now_tv)
3486 		|| !reply_info_answer_encode(qinfo, msg->rep,
3487 		*(uint16_t*)sldns_buffer_begin(buf),
3488 		sldns_buffer_read_u16_at(buf, 2),
3489 		buf, 0, 0, temp, udpsize, edns,
3490 		(int)(edns->bits&EDNS_DO), 0)) {
3491 		error_encode(buf, (LDNS_RCODE_SERVFAIL|BIT_AA), qinfo,
3492 			*(uint16_t*)sldns_buffer_begin(buf),
3493 			sldns_buffer_read_u16_at(buf, 2), edns);
3494 	}
3495 }
3496 
3497 /** encode auth error answer */
3498 static void
3499 auth_error_encode(struct query_info* qinfo, struct module_env* env,
3500 	struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3501 	struct regional* temp, int rcode)
3502 {
3503 	edns->edns_version = EDNS_ADVERTISED_VERSION;
3504 	edns->udp_size = EDNS_ADVERTISED_SIZE;
3505 	edns->ext_rcode = 0;
3506 	edns->bits &= EDNS_DO;
3507 
3508 	if(!inplace_cb_reply_local_call(env, qinfo, NULL, NULL,
3509 		rcode, edns, repinfo, temp, env->now_tv))
3510 		edns->opt_list = NULL;
3511 	error_encode(buf, rcode|BIT_AA, qinfo,
3512 		*(uint16_t*)sldns_buffer_begin(buf),
3513 		sldns_buffer_read_u16_at(buf, 2), edns);
3514 }
3515 
3516 int auth_zones_answer(struct auth_zones* az, struct module_env* env,
3517 	struct query_info* qinfo, struct edns_data* edns,
3518 	struct comm_reply* repinfo, struct sldns_buffer* buf, struct regional* temp)
3519 {
3520 	struct dns_msg* msg = NULL;
3521 	struct auth_zone* z;
3522 	int r;
3523 	int fallback = 0;
3524 
3525 	lock_rw_rdlock(&az->lock);
3526 	if(!az->have_downstream) {
3527 		/* no downstream auth zones */
3528 		lock_rw_unlock(&az->lock);
3529 		return 0;
3530 	}
3531 	if(qinfo->qtype == LDNS_RR_TYPE_DS) {
3532 		uint8_t* delname = qinfo->qname;
3533 		size_t delnamelen = qinfo->qname_len;
3534 		dname_remove_label(&delname, &delnamelen);
3535 		z = auth_zones_find_zone(az, delname, delnamelen,
3536 			qinfo->qclass);
3537 	} else {
3538 		z = auth_zones_find_zone(az, qinfo->qname, qinfo->qname_len,
3539 			qinfo->qclass);
3540 	}
3541 	if(!z) {
3542 		/* no zone above it */
3543 		lock_rw_unlock(&az->lock);
3544 		return 0;
3545 	}
3546 	lock_rw_rdlock(&z->lock);
3547 	lock_rw_unlock(&az->lock);
3548 	if(!z->for_downstream) {
3549 		lock_rw_unlock(&z->lock);
3550 		return 0;
3551 	}
3552 	if(z->zone_expired) {
3553 		if(z->fallback_enabled) {
3554 			lock_rw_unlock(&z->lock);
3555 			return 0;
3556 		}
3557 		lock_rw_unlock(&z->lock);
3558 		lock_rw_wrlock(&az->lock);
3559 		az->num_query_down++;
3560 		lock_rw_unlock(&az->lock);
3561 		auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
3562 			LDNS_RCODE_SERVFAIL);
3563 		return 1;
3564 	}
3565 
3566 	/* answer it from zone z */
3567 	r = auth_zone_generate_answer(z, qinfo, temp, &msg, &fallback);
3568 	lock_rw_unlock(&z->lock);
3569 	if(!r && fallback) {
3570 		/* fallback to regular answering (recursive) */
3571 		return 0;
3572 	}
3573 	lock_rw_wrlock(&az->lock);
3574 	az->num_query_down++;
3575 	lock_rw_unlock(&az->lock);
3576 
3577 	/* encode answer */
3578 	if(!r)
3579 		auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
3580 			LDNS_RCODE_SERVFAIL);
3581 	else	auth_answer_encode(qinfo, env, edns, repinfo, buf, temp, msg);
3582 
3583 	return 1;
3584 }
3585 
3586 int auth_zones_can_fallback(struct auth_zones* az, uint8_t* nm, size_t nmlen,
3587 	uint16_t dclass)
3588 {
3589 	int r;
3590 	struct auth_zone* z;
3591 	lock_rw_rdlock(&az->lock);
3592 	z = auth_zone_find(az, nm, nmlen, dclass);
3593 	if(!z) {
3594 		lock_rw_unlock(&az->lock);
3595 		/* no such auth zone, fallback */
3596 		return 1;
3597 	}
3598 	lock_rw_rdlock(&z->lock);
3599 	lock_rw_unlock(&az->lock);
3600 	r = z->fallback_enabled || (!z->for_upstream);
3601 	lock_rw_unlock(&z->lock);
3602 	return r;
3603 }
3604 
3605 int
3606 auth_zone_parse_notify_serial(sldns_buffer* pkt, uint32_t *serial)
3607 {
3608 	struct query_info q;
3609 	uint16_t rdlen;
3610 	memset(&q, 0, sizeof(q));
3611 	sldns_buffer_set_position(pkt, 0);
3612 	if(!query_info_parse(&q, pkt)) return 0;
3613 	if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0) return 0;
3614 	/* skip name of RR in answer section */
3615 	if(sldns_buffer_remaining(pkt) < 1) return 0;
3616 	if(pkt_dname_len(pkt) == 0) return 0;
3617 	/* check type */
3618 	if(sldns_buffer_remaining(pkt) < 10 /* type,class,ttl,rdatalen*/)
3619 		return 0;
3620 	if(sldns_buffer_read_u16(pkt) != LDNS_RR_TYPE_SOA) return 0;
3621 	sldns_buffer_skip(pkt, 2); /* class */
3622 	sldns_buffer_skip(pkt, 4); /* ttl */
3623 	rdlen = sldns_buffer_read_u16(pkt); /* rdatalen */
3624 	if(sldns_buffer_remaining(pkt) < rdlen) return 0;
3625 	if(rdlen < 22) return 0; /* bad soa length */
3626 	sldns_buffer_skip(pkt, (ssize_t)(rdlen-20));
3627 	*serial = sldns_buffer_read_u32(pkt);
3628 	/* return true when has serial in answer section */
3629 	return 1;
3630 }
3631 
3632 /** see if addr appears in the list */
3633 static int
3634 addr_in_list(struct auth_addr* list, struct sockaddr_storage* addr,
3635 	socklen_t addrlen)
3636 {
3637 	struct auth_addr* p;
3638 	for(p=list; p; p=p->next) {
3639 		if(sockaddr_cmp_addr(addr, addrlen, &p->addr, p->addrlen)==0)
3640 			return 1;
3641 	}
3642 	return 0;
3643 }
3644 
3645 /** check if an address matches a master specification (or one of its
3646  * addresses in the addr list) */
3647 static int
3648 addr_matches_master(struct auth_master* master, struct sockaddr_storage* addr,
3649 	socklen_t addrlen, struct auth_master** fromhost)
3650 {
3651 	struct sockaddr_storage a;
3652 	socklen_t alen = 0;
3653 	int net = 0;
3654 	if(addr_in_list(master->list, addr, addrlen)) {
3655 		*fromhost = master;
3656 		return 1;
3657 	}
3658 	/* compare address (but not port number, that is the destination
3659 	 * port of the master, the port number of the received notify is
3660 	 * allowed to by any port on that master) */
3661 	if(extstrtoaddr(master->host, &a, &alen) &&
3662 		sockaddr_cmp_addr(addr, addrlen, &a, alen)==0) {
3663 		*fromhost = master;
3664 		return 1;
3665 	}
3666 	/* prefixes, addr/len, like 10.0.0.0/8 */
3667 	/* not http and has a / and there is one / */
3668 	if(master->allow_notify && !master->http &&
3669 		strchr(master->host, '/') != NULL &&
3670 		strchr(master->host, '/') == strrchr(master->host, '/') &&
3671 		netblockstrtoaddr(master->host, UNBOUND_DNS_PORT, &a, &alen,
3672 		&net) && alen == addrlen) {
3673 		if(addr_in_common(addr, (addr_is_ip6(addr, addrlen)?128:32),
3674 			&a, net, alen) >= net) {
3675 			*fromhost = NULL; /* prefix does not have destination
3676 				to send the probe or transfer with */
3677 			return 1; /* matches the netblock */
3678 		}
3679 	}
3680 	return 0;
3681 }
3682 
3683 /** check access list for notifies */
3684 static int
3685 az_xfr_allowed_notify(struct auth_xfer* xfr, struct sockaddr_storage* addr,
3686 	socklen_t addrlen, struct auth_master** fromhost)
3687 {
3688 	struct auth_master* p;
3689 	for(p=xfr->allow_notify_list; p; p=p->next) {
3690 		if(addr_matches_master(p, addr, addrlen, fromhost)) {
3691 			return 1;
3692 		}
3693 	}
3694 	return 0;
3695 }
3696 
3697 /** see if the serial means the zone has to be updated, i.e. the serial
3698  * is newer than the zone serial, or we have no zone */
3699 static int
3700 xfr_serial_means_update(struct auth_xfer* xfr, uint32_t serial)
3701 {
3702 	if(!xfr->have_zone)
3703 		return 1; /* no zone, anything is better */
3704 	if(xfr->zone_expired)
3705 		return 1; /* expired, the sent serial is better than expired
3706 			data */
3707 	if(compare_serial(xfr->serial, serial) < 0)
3708 		return 1; /* our serial is smaller than the sent serial,
3709 			the data is newer, fetch it */
3710 	return 0;
3711 }
3712 
3713 /** note notify serial, updates the notify information in the xfr struct */
3714 static void
3715 xfr_note_notify_serial(struct auth_xfer* xfr, int has_serial, uint32_t serial)
3716 {
3717 	if(xfr->notify_received && xfr->notify_has_serial && has_serial) {
3718 		/* see if this serial is newer */
3719 		if(compare_serial(xfr->notify_serial, serial) < 0)
3720 			xfr->notify_serial = serial;
3721 	} else if(xfr->notify_received && xfr->notify_has_serial &&
3722 		!has_serial) {
3723 		/* remove serial, we have notify without serial */
3724 		xfr->notify_has_serial = 0;
3725 		xfr->notify_serial = 0;
3726 	} else if(xfr->notify_received && !xfr->notify_has_serial) {
3727 		/* we already have notify without serial, keep it
3728 		 * that way; no serial check when current operation
3729 		 * is done */
3730 	} else {
3731 		xfr->notify_received = 1;
3732 		xfr->notify_has_serial = has_serial;
3733 		xfr->notify_serial = serial;
3734 	}
3735 }
3736 
3737 /** process a notify serial, start new probe or note serial. xfr is locked */
3738 static void
3739 xfr_process_notify(struct auth_xfer* xfr, struct module_env* env,
3740 	int has_serial, uint32_t serial, struct auth_master* fromhost)
3741 {
3742 	/* if the serial of notify is older than we have, don't fetch
3743 	 * a zone, we already have it */
3744 	if(has_serial && !xfr_serial_means_update(xfr, serial)) {
3745 		lock_basic_unlock(&xfr->lock);
3746 		return;
3747 	}
3748 	/* start new probe with this addr src, or note serial */
3749 	if(!xfr_start_probe(xfr, env, fromhost)) {
3750 		/* not started because already in progress, note the serial */
3751 		xfr_note_notify_serial(xfr, has_serial, serial);
3752 		lock_basic_unlock(&xfr->lock);
3753 	}
3754 	/* successful end of start_probe unlocked xfr->lock */
3755 }
3756 
3757 int auth_zones_notify(struct auth_zones* az, struct module_env* env,
3758 	uint8_t* nm, size_t nmlen, uint16_t dclass,
3759 	struct sockaddr_storage* addr, socklen_t addrlen, int has_serial,
3760 	uint32_t serial, int* refused)
3761 {
3762 	struct auth_xfer* xfr;
3763 	struct auth_master* fromhost = NULL;
3764 	/* see which zone this is */
3765 	lock_rw_rdlock(&az->lock);
3766 	xfr = auth_xfer_find(az, nm, nmlen, dclass);
3767 	if(!xfr) {
3768 		lock_rw_unlock(&az->lock);
3769 		/* no such zone, refuse the notify */
3770 		*refused = 1;
3771 		return 0;
3772 	}
3773 	lock_basic_lock(&xfr->lock);
3774 	lock_rw_unlock(&az->lock);
3775 
3776 	/* check access list for notifies */
3777 	if(!az_xfr_allowed_notify(xfr, addr, addrlen, &fromhost)) {
3778 		lock_basic_unlock(&xfr->lock);
3779 		/* notify not allowed, refuse the notify */
3780 		*refused = 1;
3781 		return 0;
3782 	}
3783 
3784 	/* process the notify */
3785 	xfr_process_notify(xfr, env, has_serial, serial, fromhost);
3786 	return 1;
3787 }
3788 
3789 int auth_zones_startprobesequence(struct auth_zones* az,
3790 	struct module_env* env, uint8_t* nm, size_t nmlen, uint16_t dclass)
3791 {
3792 	struct auth_xfer* xfr;
3793 	lock_rw_rdlock(&az->lock);
3794 	xfr = auth_xfer_find(az, nm, nmlen, dclass);
3795 	if(!xfr) {
3796 		lock_rw_unlock(&az->lock);
3797 		return 0;
3798 	}
3799 	lock_basic_lock(&xfr->lock);
3800 	lock_rw_unlock(&az->lock);
3801 
3802 	xfr_process_notify(xfr, env, 0, 0, NULL);
3803 	return 1;
3804 }
3805 
3806 /** set a zone expired */
3807 static void
3808 auth_xfer_set_expired(struct auth_xfer* xfr, struct module_env* env,
3809 	int expired)
3810 {
3811 	struct auth_zone* z;
3812 
3813 	/* expire xfr */
3814 	lock_basic_lock(&xfr->lock);
3815 	xfr->zone_expired = expired;
3816 	lock_basic_unlock(&xfr->lock);
3817 
3818 	/* find auth_zone */
3819 	lock_rw_rdlock(&env->auth_zones->lock);
3820 	z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
3821 		xfr->dclass);
3822 	if(!z) {
3823 		lock_rw_unlock(&env->auth_zones->lock);
3824 		return;
3825 	}
3826 	lock_rw_wrlock(&z->lock);
3827 	lock_rw_unlock(&env->auth_zones->lock);
3828 
3829 	/* expire auth_zone */
3830 	z->zone_expired = expired;
3831 	lock_rw_unlock(&z->lock);
3832 }
3833 
3834 /** find master (from notify or probe) in list of masters */
3835 static struct auth_master*
3836 find_master_by_host(struct auth_master* list, char* host)
3837 {
3838 	struct auth_master* p;
3839 	for(p=list; p; p=p->next) {
3840 		if(strcmp(p->host, host) == 0)
3841 			return p;
3842 	}
3843 	return NULL;
3844 }
3845 
3846 /** delete the looked up auth_addrs for all the masters in the list */
3847 static void
3848 xfr_masterlist_free_addrs(struct auth_master* list)
3849 {
3850 	struct auth_master* m;
3851 	for(m=list; m; m=m->next) {
3852 		if(m->list) {
3853 			auth_free_master_addrs(m->list);
3854 			m->list = NULL;
3855 		}
3856 	}
3857 }
3858 
3859 /** copy a list of auth_addrs */
3860 static struct auth_addr*
3861 auth_addr_list_copy(struct auth_addr* source)
3862 {
3863 	struct auth_addr* list = NULL, *last = NULL;
3864 	struct auth_addr* p;
3865 	for(p=source; p; p=p->next) {
3866 		struct auth_addr* a = (struct auth_addr*)memdup(p, sizeof(*p));
3867 		if(!a) {
3868 			log_err("malloc failure");
3869 			auth_free_master_addrs(list);
3870 			return NULL;
3871 		}
3872 		a->next = NULL;
3873 		if(last) last->next = a;
3874 		if(!list) list = a;
3875 		last = a;
3876 	}
3877 	return list;
3878 }
3879 
3880 /** copy a master to a new structure, NULL on alloc failure */
3881 static struct auth_master*
3882 auth_master_copy(struct auth_master* o)
3883 {
3884 	struct auth_master* m;
3885 	if(!o) return NULL;
3886 	m = (struct auth_master*)memdup(o, sizeof(*o));
3887 	if(!m) {
3888 		log_err("malloc failure");
3889 		return NULL;
3890 	}
3891 	m->next = NULL;
3892 	if(m->host) {
3893 		m->host = strdup(m->host);
3894 		if(!m->host) {
3895 			free(m);
3896 			log_err("malloc failure");
3897 			return NULL;
3898 		}
3899 	}
3900 	if(m->file) {
3901 		m->file = strdup(m->file);
3902 		if(!m->file) {
3903 			free(m->host);
3904 			free(m);
3905 			log_err("malloc failure");
3906 			return NULL;
3907 		}
3908 	}
3909 	if(m->list) {
3910 		m->list = auth_addr_list_copy(m->list);
3911 		if(!m->list) {
3912 			free(m->file);
3913 			free(m->host);
3914 			free(m);
3915 			return NULL;
3916 		}
3917 	}
3918 	return m;
3919 }
3920 
3921 /** copy the master addresses from the task_probe lookups to the allow_notify
3922  * list of masters */
3923 static void
3924 probe_copy_masters_for_allow_notify(struct auth_xfer* xfr)
3925 {
3926 	struct auth_master* list = NULL, *last = NULL;
3927 	struct auth_master* p;
3928 	/* build up new list with copies */
3929 	for(p = xfr->task_probe->masters; p; p=p->next) {
3930 		struct auth_master* m = auth_master_copy(p);
3931 		if(!m) {
3932 			auth_free_masters(list);
3933 			/* failed because of malloc failure, use old list */
3934 			return;
3935 		}
3936 		m->next = NULL;
3937 		if(last) last->next = m;
3938 		if(!list) list = m;
3939 		last = m;
3940 	}
3941 	/* success, replace list */
3942 	auth_free_masters(xfr->allow_notify_list);
3943 	xfr->allow_notify_list = list;
3944 }
3945 
3946 /** start the lookups for task_transfer */
3947 static void
3948 xfr_transfer_start_lookups(struct auth_xfer* xfr)
3949 {
3950 	/* delete all the looked up addresses in the list */
3951 	xfr->task_transfer->scan_addr = NULL;
3952 	xfr_masterlist_free_addrs(xfr->task_transfer->masters);
3953 
3954 	/* start lookup at the first master */
3955 	xfr->task_transfer->lookup_target = xfr->task_transfer->masters;
3956 	xfr->task_transfer->lookup_aaaa = 0;
3957 }
3958 
3959 /** move to the next lookup of hostname for task_transfer */
3960 static void
3961 xfr_transfer_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
3962 {
3963 	if(!xfr->task_transfer->lookup_target)
3964 		return; /* already at end of list */
3965 	if(!xfr->task_transfer->lookup_aaaa && env->cfg->do_ip6) {
3966 		/* move to lookup AAAA */
3967 		xfr->task_transfer->lookup_aaaa = 1;
3968 		return;
3969 	}
3970 	xfr->task_transfer->lookup_target =
3971 		xfr->task_transfer->lookup_target->next;
3972 	xfr->task_transfer->lookup_aaaa = 0;
3973 	if(!env->cfg->do_ip4 && xfr->task_transfer->lookup_target!=NULL)
3974 		xfr->task_transfer->lookup_aaaa = 1;
3975 }
3976 
3977 /** start the lookups for task_probe */
3978 static void
3979 xfr_probe_start_lookups(struct auth_xfer* xfr)
3980 {
3981 	/* delete all the looked up addresses in the list */
3982 	xfr->task_probe->scan_addr = NULL;
3983 	xfr_masterlist_free_addrs(xfr->task_probe->masters);
3984 
3985 	/* start lookup at the first master */
3986 	xfr->task_probe->lookup_target = xfr->task_probe->masters;
3987 	xfr->task_probe->lookup_aaaa = 0;
3988 }
3989 
3990 /** move to the next lookup of hostname for task_probe */
3991 static void
3992 xfr_probe_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
3993 {
3994 	if(!xfr->task_probe->lookup_target)
3995 		return; /* already at end of list */
3996 	if(!xfr->task_probe->lookup_aaaa && env->cfg->do_ip6) {
3997 		/* move to lookup AAAA */
3998 		xfr->task_probe->lookup_aaaa = 1;
3999 		return;
4000 	}
4001 	xfr->task_probe->lookup_target = xfr->task_probe->lookup_target->next;
4002 	xfr->task_probe->lookup_aaaa = 0;
4003 	if(!env->cfg->do_ip4 && xfr->task_probe->lookup_target!=NULL)
4004 		xfr->task_probe->lookup_aaaa = 1;
4005 }
4006 
4007 /** start the iteration of the task_transfer list of masters */
4008 static void
4009 xfr_transfer_start_list(struct auth_xfer* xfr, struct auth_master* spec)
4010 {
4011 	if(spec) {
4012 		xfr->task_transfer->scan_specific = find_master_by_host(
4013 			xfr->task_transfer->masters, spec->host);
4014 		if(xfr->task_transfer->scan_specific) {
4015 			xfr->task_transfer->scan_target = NULL;
4016 			xfr->task_transfer->scan_addr = NULL;
4017 			if(xfr->task_transfer->scan_specific->list)
4018 				xfr->task_transfer->scan_addr =
4019 					xfr->task_transfer->scan_specific->list;
4020 			return;
4021 		}
4022 	}
4023 	/* no specific (notified) host to scan */
4024 	xfr->task_transfer->scan_specific = NULL;
4025 	xfr->task_transfer->scan_addr = NULL;
4026 	/* pick up first scan target */
4027 	xfr->task_transfer->scan_target = xfr->task_transfer->masters;
4028 	if(xfr->task_transfer->scan_target && xfr->task_transfer->
4029 		scan_target->list)
4030 		xfr->task_transfer->scan_addr =
4031 			xfr->task_transfer->scan_target->list;
4032 }
4033 
4034 /** start the iteration of the task_probe list of masters */
4035 static void
4036 xfr_probe_start_list(struct auth_xfer* xfr, struct auth_master* spec)
4037 {
4038 	if(spec) {
4039 		xfr->task_probe->scan_specific = find_master_by_host(
4040 			xfr->task_probe->masters, spec->host);
4041 		if(xfr->task_probe->scan_specific) {
4042 			xfr->task_probe->scan_target = NULL;
4043 			xfr->task_probe->scan_addr = NULL;
4044 			if(xfr->task_probe->scan_specific->list)
4045 				xfr->task_probe->scan_addr =
4046 					xfr->task_probe->scan_specific->list;
4047 			return;
4048 		}
4049 	}
4050 	/* no specific (notified) host to scan */
4051 	xfr->task_probe->scan_specific = NULL;
4052 	xfr->task_probe->scan_addr = NULL;
4053 	/* pick up first scan target */
4054 	xfr->task_probe->scan_target = xfr->task_probe->masters;
4055 	if(xfr->task_probe->scan_target && xfr->task_probe->scan_target->list)
4056 		xfr->task_probe->scan_addr =
4057 			xfr->task_probe->scan_target->list;
4058 }
4059 
4060 /** pick up the master that is being scanned right now, task_transfer */
4061 static struct auth_master*
4062 xfr_transfer_current_master(struct auth_xfer* xfr)
4063 {
4064 	if(xfr->task_transfer->scan_specific)
4065 		return xfr->task_transfer->scan_specific;
4066 	return xfr->task_transfer->scan_target;
4067 }
4068 
4069 /** pick up the master that is being scanned right now, task_probe */
4070 static struct auth_master*
4071 xfr_probe_current_master(struct auth_xfer* xfr)
4072 {
4073 	if(xfr->task_probe->scan_specific)
4074 		return xfr->task_probe->scan_specific;
4075 	return xfr->task_probe->scan_target;
4076 }
4077 
4078 /** true if at end of list, task_transfer */
4079 static int
4080 xfr_transfer_end_of_list(struct auth_xfer* xfr)
4081 {
4082 	return !xfr->task_transfer->scan_specific &&
4083 		!xfr->task_transfer->scan_target;
4084 }
4085 
4086 /** true if at end of list, task_probe */
4087 static int
4088 xfr_probe_end_of_list(struct auth_xfer* xfr)
4089 {
4090 	return !xfr->task_probe->scan_specific && !xfr->task_probe->scan_target;
4091 }
4092 
4093 /** move to next master in list, task_transfer */
4094 static void
4095 xfr_transfer_nextmaster(struct auth_xfer* xfr)
4096 {
4097 	if(!xfr->task_transfer->scan_specific &&
4098 		!xfr->task_transfer->scan_target)
4099 		return;
4100 	if(xfr->task_transfer->scan_addr) {
4101 		xfr->task_transfer->scan_addr =
4102 			xfr->task_transfer->scan_addr->next;
4103 		if(xfr->task_transfer->scan_addr)
4104 			return;
4105 	}
4106 	if(xfr->task_transfer->scan_specific) {
4107 		xfr->task_transfer->scan_specific = NULL;
4108 		xfr->task_transfer->scan_target = xfr->task_transfer->masters;
4109 		if(xfr->task_transfer->scan_target && xfr->task_transfer->
4110 			scan_target->list)
4111 			xfr->task_transfer->scan_addr =
4112 				xfr->task_transfer->scan_target->list;
4113 		return;
4114 	}
4115 	if(!xfr->task_transfer->scan_target)
4116 		return;
4117 	xfr->task_transfer->scan_target = xfr->task_transfer->scan_target->next;
4118 	if(xfr->task_transfer->scan_target && xfr->task_transfer->
4119 		scan_target->list)
4120 		xfr->task_transfer->scan_addr =
4121 			xfr->task_transfer->scan_target->list;
4122 	return;
4123 }
4124 
4125 /** move to next master in list, task_probe */
4126 static void
4127 xfr_probe_nextmaster(struct auth_xfer* xfr)
4128 {
4129 	if(!xfr->task_probe->scan_specific && !xfr->task_probe->scan_target)
4130 		return;
4131 	if(xfr->task_probe->scan_addr) {
4132 		xfr->task_probe->scan_addr = xfr->task_probe->scan_addr->next;
4133 		if(xfr->task_probe->scan_addr)
4134 			return;
4135 	}
4136 	if(xfr->task_probe->scan_specific) {
4137 		xfr->task_probe->scan_specific = NULL;
4138 		xfr->task_probe->scan_target = xfr->task_probe->masters;
4139 		if(xfr->task_probe->scan_target && xfr->task_probe->
4140 			scan_target->list)
4141 			xfr->task_probe->scan_addr =
4142 				xfr->task_probe->scan_target->list;
4143 		return;
4144 	}
4145 	if(!xfr->task_probe->scan_target)
4146 		return;
4147 	xfr->task_probe->scan_target = xfr->task_probe->scan_target->next;
4148 	if(xfr->task_probe->scan_target && xfr->task_probe->
4149 		scan_target->list)
4150 		xfr->task_probe->scan_addr =
4151 			xfr->task_probe->scan_target->list;
4152 	return;
4153 }
4154 
4155 /** create SOA probe packet for xfr */
4156 static void
4157 xfr_create_soa_probe_packet(struct auth_xfer* xfr, sldns_buffer* buf,
4158 	uint16_t id)
4159 {
4160 	struct query_info qinfo;
4161 
4162 	memset(&qinfo, 0, sizeof(qinfo));
4163 	qinfo.qname = xfr->name;
4164 	qinfo.qname_len = xfr->namelen;
4165 	qinfo.qtype = LDNS_RR_TYPE_SOA;
4166 	qinfo.qclass = xfr->dclass;
4167 	qinfo_query_encode(buf, &qinfo);
4168 	sldns_buffer_write_u16_at(buf, 0, id);
4169 }
4170 
4171 /** create IXFR/AXFR packet for xfr */
4172 static void
4173 xfr_create_ixfr_packet(struct auth_xfer* xfr, sldns_buffer* buf, uint16_t id,
4174 	struct auth_master* master)
4175 {
4176 	struct query_info qinfo;
4177 	uint32_t serial;
4178 	int have_zone;
4179 	have_zone = xfr->have_zone;
4180 	serial = xfr->serial;
4181 
4182 	memset(&qinfo, 0, sizeof(qinfo));
4183 	qinfo.qname = xfr->name;
4184 	qinfo.qname_len = xfr->namelen;
4185 	xfr->task_transfer->got_xfr_serial = 0;
4186 	xfr->task_transfer->rr_scan_num = 0;
4187 	xfr->task_transfer->incoming_xfr_serial = 0;
4188 	xfr->task_transfer->on_ixfr_is_axfr = 0;
4189 	xfr->task_transfer->on_ixfr = 1;
4190 	qinfo.qtype = LDNS_RR_TYPE_IXFR;
4191 	if(!have_zone || xfr->task_transfer->ixfr_fail || !master->ixfr) {
4192 		qinfo.qtype = LDNS_RR_TYPE_AXFR;
4193 		xfr->task_transfer->ixfr_fail = 0;
4194 		xfr->task_transfer->on_ixfr = 0;
4195 	}
4196 
4197 	qinfo.qclass = xfr->dclass;
4198 	qinfo_query_encode(buf, &qinfo);
4199 	sldns_buffer_write_u16_at(buf, 0, id);
4200 
4201 	/* append serial for IXFR */
4202 	if(qinfo.qtype == LDNS_RR_TYPE_IXFR) {
4203 		size_t end = sldns_buffer_limit(buf);
4204 		sldns_buffer_clear(buf);
4205 		sldns_buffer_set_position(buf, end);
4206 		/* auth section count 1 */
4207 		sldns_buffer_write_u16_at(buf, LDNS_NSCOUNT_OFF, 1);
4208 		/* write SOA */
4209 		sldns_buffer_write_u8(buf, 0xC0); /* compressed ptr to qname */
4210 		sldns_buffer_write_u8(buf, 0x0C);
4211 		sldns_buffer_write_u16(buf, LDNS_RR_TYPE_SOA);
4212 		sldns_buffer_write_u16(buf, qinfo.qclass);
4213 		sldns_buffer_write_u32(buf, 0); /* ttl */
4214 		sldns_buffer_write_u16(buf, 22); /* rdata length */
4215 		sldns_buffer_write_u8(buf, 0); /* . */
4216 		sldns_buffer_write_u8(buf, 0); /* . */
4217 		sldns_buffer_write_u32(buf, serial); /* serial */
4218 		sldns_buffer_write_u32(buf, 0); /* refresh */
4219 		sldns_buffer_write_u32(buf, 0); /* retry */
4220 		sldns_buffer_write_u32(buf, 0); /* expire */
4221 		sldns_buffer_write_u32(buf, 0); /* minimum */
4222 		sldns_buffer_flip(buf);
4223 	}
4224 }
4225 
4226 /** check if returned packet is OK */
4227 static int
4228 check_packet_ok(sldns_buffer* pkt, uint16_t qtype, struct auth_xfer* xfr,
4229 	uint32_t* serial)
4230 {
4231 	/* parse to see if packet worked, valid reply */
4232 
4233 	/* check serial number of SOA */
4234 	if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE)
4235 		return 0;
4236 
4237 	/* check ID */
4238 	if(LDNS_ID_WIRE(sldns_buffer_begin(pkt)) != xfr->task_probe->id)
4239 		return 0;
4240 
4241 	/* check flag bits and rcode */
4242 	if(!LDNS_QR_WIRE(sldns_buffer_begin(pkt)))
4243 		return 0;
4244 	if(LDNS_OPCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_PACKET_QUERY)
4245 		return 0;
4246 	if(LDNS_RCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_RCODE_NOERROR)
4247 		return 0;
4248 
4249 	/* check qname */
4250 	if(LDNS_QDCOUNT(sldns_buffer_begin(pkt)) != 1)
4251 		return 0;
4252 	sldns_buffer_skip(pkt, LDNS_HEADER_SIZE);
4253 	if(sldns_buffer_remaining(pkt) < xfr->namelen)
4254 		return 0;
4255 	if(query_dname_compare(sldns_buffer_current(pkt), xfr->name) != 0)
4256 		return 0;
4257 	sldns_buffer_skip(pkt, (ssize_t)xfr->namelen);
4258 
4259 	/* check qtype, qclass */
4260 	if(sldns_buffer_remaining(pkt) < 4)
4261 		return 0;
4262 	if(sldns_buffer_read_u16(pkt) != qtype)
4263 		return 0;
4264 	if(sldns_buffer_read_u16(pkt) != xfr->dclass)
4265 		return 0;
4266 
4267 	if(serial) {
4268 		uint16_t rdlen;
4269 		/* read serial number, from answer section SOA */
4270 		if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0)
4271 			return 0;
4272 		/* read from first record SOA record */
4273 		if(sldns_buffer_remaining(pkt) < 1)
4274 			return 0;
4275 		if(dname_pkt_compare(pkt, sldns_buffer_current(pkt),
4276 			xfr->name) != 0)
4277 			return 0;
4278 		if(!pkt_dname_len(pkt))
4279 			return 0;
4280 		/* type, class, ttl, rdatalen */
4281 		if(sldns_buffer_remaining(pkt) < 4+4+2)
4282 			return 0;
4283 		if(sldns_buffer_read_u16(pkt) != qtype)
4284 			return 0;
4285 		if(sldns_buffer_read_u16(pkt) != xfr->dclass)
4286 			return 0;
4287 		sldns_buffer_skip(pkt, 4); /* ttl */
4288 		rdlen = sldns_buffer_read_u16(pkt);
4289 		if(sldns_buffer_remaining(pkt) < rdlen)
4290 			return 0;
4291 		if(sldns_buffer_remaining(pkt) < 1)
4292 			return 0;
4293 		if(!pkt_dname_len(pkt)) /* soa name */
4294 			return 0;
4295 		if(sldns_buffer_remaining(pkt) < 1)
4296 			return 0;
4297 		if(!pkt_dname_len(pkt)) /* soa name */
4298 			return 0;
4299 		if(sldns_buffer_remaining(pkt) < 20)
4300 			return 0;
4301 		*serial = sldns_buffer_read_u32(pkt);
4302 	}
4303 	return 1;
4304 }
4305 
4306 /** read one line from chunks into buffer at current position */
4307 static int
4308 chunkline_get_line(struct auth_chunk** chunk, size_t* chunk_pos,
4309 	sldns_buffer* buf)
4310 {
4311 	int readsome = 0;
4312 	while(*chunk) {
4313 		/* more text in this chunk? */
4314 		if(*chunk_pos < (*chunk)->len) {
4315 			readsome = 1;
4316 			while(*chunk_pos < (*chunk)->len) {
4317 				char c = (char)((*chunk)->data[*chunk_pos]);
4318 				(*chunk_pos)++;
4319 				if(sldns_buffer_remaining(buf) < 2) {
4320 					/* buffer too short */
4321 					verbose(VERB_ALGO, "http chunkline, "
4322 						"line too long");
4323 					return 0;
4324 				}
4325 				sldns_buffer_write_u8(buf, (uint8_t)c);
4326 				if(c == '\n') {
4327 					/* we are done */
4328 					return 1;
4329 				}
4330 			}
4331 		}
4332 		/* move to next chunk */
4333 		*chunk = (*chunk)->next;
4334 		*chunk_pos = 0;
4335 	}
4336 	/* no more text */
4337 	if(readsome) return 1;
4338 	return 0;
4339 }
4340 
4341 /** count number of open and closed parenthesis in a chunkline */
4342 static int
4343 chunkline_count_parens(sldns_buffer* buf, size_t start)
4344 {
4345 	size_t end = sldns_buffer_position(buf);
4346 	size_t i;
4347 	int count = 0;
4348 	int squote = 0, dquote = 0;
4349 	for(i=start; i<end; i++) {
4350 		char c = (char)sldns_buffer_read_u8_at(buf, i);
4351 		if(squote && c != '\'') continue;
4352 		if(dquote && c != '"') continue;
4353 		if(c == '"')
4354 			dquote = !dquote; /* skip quoted part */
4355 		else if(c == '\'')
4356 			squote = !squote; /* skip quoted part */
4357 		else if(c == '(')
4358 			count ++;
4359 		else if(c == ')')
4360 			count --;
4361 		else if(c == ';') {
4362 			/* rest is a comment */
4363 			return count;
4364 		}
4365 	}
4366 	return count;
4367 }
4368 
4369 /** remove trailing ;... comment from a line in the chunkline buffer */
4370 static void
4371 chunkline_remove_trailcomment(sldns_buffer* buf, size_t start)
4372 {
4373 	size_t end = sldns_buffer_position(buf);
4374 	size_t i;
4375 	int squote = 0, dquote = 0;
4376 	for(i=start; i<end; i++) {
4377 		char c = (char)sldns_buffer_read_u8_at(buf, i);
4378 		if(squote && c != '\'') continue;
4379 		if(dquote && c != '"') continue;
4380 		if(c == '"')
4381 			dquote = !dquote; /* skip quoted part */
4382 		else if(c == '\'')
4383 			squote = !squote; /* skip quoted part */
4384 		else if(c == ';') {
4385 			/* rest is a comment */
4386 			sldns_buffer_set_position(buf, i);
4387 			return;
4388 		}
4389 	}
4390 	/* nothing to remove */
4391 }
4392 
4393 /** see if a chunkline is a comment line (or empty line) */
4394 static int
4395 chunkline_is_comment_line_or_empty(sldns_buffer* buf)
4396 {
4397 	size_t i, end = sldns_buffer_limit(buf);
4398 	for(i=0; i<end; i++) {
4399 		char c = (char)sldns_buffer_read_u8_at(buf, i);
4400 		if(c == ';')
4401 			return 1; /* comment */
4402 		else if(c != ' ' && c != '\t' && c != '\r' && c != '\n')
4403 			return 0; /* not a comment */
4404 	}
4405 	return 1; /* empty */
4406 }
4407 
4408 /** find a line with ( ) collated */
4409 static int
4410 chunkline_get_line_collated(struct auth_chunk** chunk, size_t* chunk_pos,
4411 	sldns_buffer* buf)
4412 {
4413 	size_t pos;
4414 	int parens = 0;
4415 	sldns_buffer_clear(buf);
4416 	pos = sldns_buffer_position(buf);
4417 	if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4418 		if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4419 			sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4420 		else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4421 		sldns_buffer_flip(buf);
4422 		return 0;
4423 	}
4424 	parens += chunkline_count_parens(buf, pos);
4425 	while(parens > 0) {
4426 		chunkline_remove_trailcomment(buf, pos);
4427 		pos = sldns_buffer_position(buf);
4428 		if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4429 			if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4430 				sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4431 			else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4432 			sldns_buffer_flip(buf);
4433 			return 0;
4434 		}
4435 		parens += chunkline_count_parens(buf, pos);
4436 	}
4437 
4438 	if(sldns_buffer_remaining(buf) < 1) {
4439 		verbose(VERB_ALGO, "http chunkline: "
4440 			"line too long");
4441 		return 0;
4442 	}
4443 	sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4444 	sldns_buffer_flip(buf);
4445 	return 1;
4446 }
4447 
4448 /** process $ORIGIN for http */
4449 static int
4450 http_parse_origin(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4451 {
4452 	char* line = (char*)sldns_buffer_begin(buf);
4453 	if(strncmp(line, "$ORIGIN", 7) == 0 &&
4454 		isspace((unsigned char)line[7])) {
4455 		int s;
4456 		pstate->origin_len = sizeof(pstate->origin);
4457 		s = sldns_str2wire_dname_buf(sldns_strip_ws(line+8),
4458 			pstate->origin, &pstate->origin_len);
4459 		if(s) pstate->origin_len = 0;
4460 		return 1;
4461 	}
4462 	return 0;
4463 }
4464 
4465 /** process $TTL for http */
4466 static int
4467 http_parse_ttl(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4468 {
4469 	char* line = (char*)sldns_buffer_begin(buf);
4470 	if(strncmp(line, "$TTL", 4) == 0 &&
4471 		isspace((unsigned char)line[4])) {
4472 		const char* end = NULL;
4473 		pstate->default_ttl = sldns_str2period(
4474 			sldns_strip_ws(line+5), &end);
4475 		return 1;
4476 	}
4477 	return 0;
4478 }
4479 
4480 /** find noncomment RR line in chunks, collates lines if ( ) format */
4481 static int
4482 chunkline_non_comment_RR(struct auth_chunk** chunk, size_t* chunk_pos,
4483 	sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4484 {
4485 	while(chunkline_get_line_collated(chunk, chunk_pos, buf)) {
4486 		if(chunkline_is_comment_line_or_empty(buf)) {
4487 			/* a comment, go to next line */
4488 			continue;
4489 		}
4490 		if(http_parse_origin(buf, pstate)) {
4491 			continue; /* $ORIGIN has been handled */
4492 		}
4493 		if(http_parse_ttl(buf, pstate)) {
4494 			continue; /* $TTL has been handled */
4495 		}
4496 		return 1;
4497 	}
4498 	/* no noncomments, fail */
4499 	return 0;
4500 }
4501 
4502 /** check syntax of chunklist zonefile, parse first RR, return false on
4503  * failure and return a string in the scratch buffer (first RR string)
4504  * on failure. */
4505 static int
4506 http_zonefile_syntax_check(struct auth_xfer* xfr, sldns_buffer* buf)
4507 {
4508 	uint8_t rr[LDNS_RR_BUF_SIZE];
4509 	size_t rr_len, dname_len = 0;
4510 	struct sldns_file_parse_state pstate;
4511 	struct auth_chunk* chunk;
4512 	size_t chunk_pos;
4513 	int e;
4514 	memset(&pstate, 0, sizeof(pstate));
4515 	pstate.default_ttl = 3600;
4516 	if(xfr->namelen < sizeof(pstate.origin)) {
4517 		pstate.origin_len = xfr->namelen;
4518 		memmove(pstate.origin, xfr->name, xfr->namelen);
4519 	}
4520 	chunk = xfr->task_transfer->chunks_first;
4521 	chunk_pos = 0;
4522 	if(!chunkline_non_comment_RR(&chunk, &chunk_pos, buf, &pstate)) {
4523 		return 0;
4524 	}
4525 	rr_len = sizeof(rr);
4526 	e=sldns_str2wire_rr_buf((char*)sldns_buffer_begin(buf), rr, &rr_len,
4527 		&dname_len, pstate.default_ttl,
4528 		pstate.origin_len?pstate.origin:NULL, pstate.origin_len,
4529 		pstate.prev_rr_len?pstate.prev_rr:NULL, pstate.prev_rr_len);
4530 	if(e != 0) {
4531 		log_err("parse failure on first RR[%d]: %s",
4532 			LDNS_WIREPARSE_OFFSET(e),
4533 			sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)));
4534 		return 0;
4535 	}
4536 	/* check that class is correct */
4537 	if(sldns_wirerr_get_class(rr, rr_len, dname_len) != xfr->dclass) {
4538 		log_err("parse failure: first record in downloaded zonefile "
4539 			"from wrong RR class");
4540 		return 0;
4541 	}
4542 	return 1;
4543 }
4544 
4545 /** sum sizes of chunklist */
4546 static size_t
4547 chunklist_sum(struct auth_chunk* list)
4548 {
4549 	struct auth_chunk* p;
4550 	size_t s = 0;
4551 	for(p=list; p; p=p->next) {
4552 		s += p->len;
4553 	}
4554 	return s;
4555 }
4556 
4557 /** remove newlines from collated line */
4558 static void
4559 chunkline_newline_removal(sldns_buffer* buf)
4560 {
4561 	size_t i, end=sldns_buffer_limit(buf);
4562 	for(i=0; i<end; i++) {
4563 		char c = (char)sldns_buffer_read_u8_at(buf, i);
4564 		if(c == '\n' && i==end-1) {
4565 			sldns_buffer_write_u8_at(buf, i, 0);
4566 			sldns_buffer_set_limit(buf, end-1);
4567 			return;
4568 		}
4569 		if(c == '\n')
4570 			sldns_buffer_write_u8_at(buf, i, (uint8_t)' ');
4571 	}
4572 }
4573 
4574 /** for http download, parse and add RR to zone */
4575 static int
4576 http_parse_add_rr(struct auth_xfer* xfr, struct auth_zone* z,
4577 	sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4578 {
4579 	uint8_t rr[LDNS_RR_BUF_SIZE];
4580 	size_t rr_len, dname_len = 0;
4581 	int e;
4582 	char* line = (char*)sldns_buffer_begin(buf);
4583 	rr_len = sizeof(rr);
4584 	e = sldns_str2wire_rr_buf(line, rr, &rr_len, &dname_len,
4585 		pstate->default_ttl,
4586 		pstate->origin_len?pstate->origin:NULL, pstate->origin_len,
4587 		pstate->prev_rr_len?pstate->prev_rr:NULL, pstate->prev_rr_len);
4588 	if(e != 0) {
4589 		log_err("%s/%s parse failure RR[%d]: %s in '%s'",
4590 			xfr->task_transfer->master->host,
4591 			xfr->task_transfer->master->file,
4592 			LDNS_WIREPARSE_OFFSET(e),
4593 			sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)),
4594 			line);
4595 		return 0;
4596 	}
4597 	if(rr_len == 0)
4598 		return 1; /* empty line or so */
4599 
4600 	/* set prev */
4601 	if(dname_len < sizeof(pstate->prev_rr)) {
4602 		memmove(pstate->prev_rr, rr, dname_len);
4603 		pstate->prev_rr_len = dname_len;
4604 	}
4605 
4606 	return az_insert_rr(z, rr, rr_len, dname_len, NULL);
4607 }
4608 
4609 /** RR list iterator, returns RRs from answer section one by one from the
4610  * dns packets in the chunklist */
4611 static void
4612 chunk_rrlist_start(struct auth_xfer* xfr, struct auth_chunk** rr_chunk,
4613 	int* rr_num, size_t* rr_pos)
4614 {
4615 	*rr_chunk = xfr->task_transfer->chunks_first;
4616 	*rr_num = 0;
4617 	*rr_pos = 0;
4618 }
4619 
4620 /** RR list iterator, see if we are at the end of the list */
4621 static int
4622 chunk_rrlist_end(struct auth_chunk* rr_chunk, int rr_num)
4623 {
4624 	while(rr_chunk) {
4625 		if(rr_chunk->len < LDNS_HEADER_SIZE)
4626 			return 1;
4627 		if(rr_num < (int)LDNS_ANCOUNT(rr_chunk->data))
4628 			return 0;
4629 		/* no more RRs in this chunk */
4630 		/* continue with next chunk, see if it has RRs */
4631 		rr_chunk = rr_chunk->next;
4632 		rr_num = 0;
4633 	}
4634 	return 1;
4635 }
4636 
4637 /** RR list iterator, move to next RR */
4638 static void
4639 chunk_rrlist_gonext(struct auth_chunk** rr_chunk, int* rr_num,
4640 	size_t* rr_pos, size_t rr_nextpos)
4641 {
4642 	/* already at end of chunks? */
4643 	if(!*rr_chunk)
4644 		return;
4645 	/* move within this chunk */
4646 	if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4647 		(*rr_num)+1 < (int)LDNS_ANCOUNT((*rr_chunk)->data)) {
4648 		(*rr_num) += 1;
4649 		*rr_pos = rr_nextpos;
4650 		return;
4651 	}
4652 	/* no more RRs in this chunk */
4653 	/* continue with next chunk, see if it has RRs */
4654 	if(*rr_chunk)
4655 		*rr_chunk = (*rr_chunk)->next;
4656 	while(*rr_chunk) {
4657 		*rr_num = 0;
4658 		*rr_pos = 0;
4659 		if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4660 			LDNS_ANCOUNT((*rr_chunk)->data) > 0) {
4661 			return;
4662 		}
4663 		*rr_chunk = (*rr_chunk)->next;
4664 	}
4665 }
4666 
4667 /** RR iterator, get current RR information, false on parse error */
4668 static int
4669 chunk_rrlist_get_current(struct auth_chunk* rr_chunk, int rr_num,
4670 	size_t rr_pos, uint8_t** rr_dname, uint16_t* rr_type,
4671 	uint16_t* rr_class, uint32_t* rr_ttl, uint16_t* rr_rdlen,
4672 	uint8_t** rr_rdata, size_t* rr_nextpos)
4673 {
4674 	sldns_buffer pkt;
4675 	/* integrity checks on position */
4676 	if(!rr_chunk) return 0;
4677 	if(rr_chunk->len < LDNS_HEADER_SIZE) return 0;
4678 	if(rr_num >= (int)LDNS_ANCOUNT(rr_chunk->data)) return 0;
4679 	if(rr_pos >= rr_chunk->len) return 0;
4680 
4681 	/* fetch rr information */
4682 	sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4683 	if(rr_pos == 0) {
4684 		size_t i;
4685 		/* skip question section */
4686 		sldns_buffer_set_position(&pkt, LDNS_HEADER_SIZE);
4687 		for(i=0; i<LDNS_QDCOUNT(rr_chunk->data); i++) {
4688 			if(pkt_dname_len(&pkt) == 0) return 0;
4689 			if(sldns_buffer_remaining(&pkt) < 4) return 0;
4690 			sldns_buffer_skip(&pkt, 4); /* type and class */
4691 		}
4692 	} else	{
4693 		sldns_buffer_set_position(&pkt, rr_pos);
4694 	}
4695 	*rr_dname = sldns_buffer_current(&pkt);
4696 	if(pkt_dname_len(&pkt) == 0) return 0;
4697 	if(sldns_buffer_remaining(&pkt) < 10) return 0;
4698 	*rr_type = sldns_buffer_read_u16(&pkt);
4699 	*rr_class = sldns_buffer_read_u16(&pkt);
4700 	*rr_ttl = sldns_buffer_read_u32(&pkt);
4701 	*rr_rdlen = sldns_buffer_read_u16(&pkt);
4702 	if(sldns_buffer_remaining(&pkt) < (*rr_rdlen)) return 0;
4703 	*rr_rdata = sldns_buffer_current(&pkt);
4704 	sldns_buffer_skip(&pkt, (ssize_t)(*rr_rdlen));
4705 	*rr_nextpos = sldns_buffer_position(&pkt);
4706 	return 1;
4707 }
4708 
4709 /** print log message where we are in parsing the zone transfer */
4710 static void
4711 log_rrlist_position(const char* label, struct auth_chunk* rr_chunk,
4712 	uint8_t* rr_dname, uint16_t rr_type, size_t rr_counter)
4713 {
4714 	sldns_buffer pkt;
4715 	size_t dlen;
4716 	uint8_t buf[256];
4717 	char str[256];
4718 	char typestr[32];
4719 	sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4720 	sldns_buffer_set_position(&pkt, (size_t)(rr_dname -
4721 		sldns_buffer_begin(&pkt)));
4722 	if((dlen=pkt_dname_len(&pkt)) == 0) return;
4723 	if(dlen >= sizeof(buf)) return;
4724 	dname_pkt_copy(&pkt, buf, rr_dname);
4725 	dname_str(buf, str);
4726 	(void)sldns_wire2str_type_buf(rr_type, typestr, sizeof(typestr));
4727 	verbose(VERB_ALGO, "%s at[%d] %s %s", label, (int)rr_counter,
4728 		str, typestr);
4729 }
4730 
4731 /** check that start serial is OK for ixfr. we are at rr_counter == 0,
4732  * and we are going to check rr_counter == 1 (has to be type SOA) serial */
4733 static int
4734 ixfr_start_serial(struct auth_chunk* rr_chunk, int rr_num, size_t rr_pos,
4735 	uint8_t* rr_dname, uint16_t rr_type, uint16_t rr_class,
4736 	uint32_t rr_ttl, uint16_t rr_rdlen, uint8_t* rr_rdata,
4737 	size_t rr_nextpos, uint32_t transfer_serial, uint32_t xfr_serial)
4738 {
4739 	uint32_t startserial;
4740 	/* move forward on RR */
4741 	chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4742 	if(chunk_rrlist_end(rr_chunk, rr_num)) {
4743 		/* no second SOA */
4744 		verbose(VERB_OPS, "IXFR has no second SOA record");
4745 		return 0;
4746 	}
4747 	if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4748 		&rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4749 		&rr_rdata, &rr_nextpos)) {
4750 		verbose(VERB_OPS, "IXFR cannot parse second SOA record");
4751 		/* failed to parse RR */
4752 		return 0;
4753 	}
4754 	if(rr_type != LDNS_RR_TYPE_SOA) {
4755 		verbose(VERB_OPS, "IXFR second record is not type SOA");
4756 		return 0;
4757 	}
4758 	if(rr_rdlen < 22) {
4759 		verbose(VERB_OPS, "IXFR, second SOA has short rdlength");
4760 		return 0; /* bad SOA rdlen */
4761 	}
4762 	startserial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4763 	if(startserial == transfer_serial) {
4764 		/* empty AXFR, not an IXFR */
4765 		verbose(VERB_OPS, "IXFR second serial same as first");
4766 		return 0;
4767 	}
4768 	if(startserial != xfr_serial) {
4769 		/* wrong start serial, it does not match the serial in
4770 		 * memory */
4771 		verbose(VERB_OPS, "IXFR is from serial %u to %u but %u "
4772 			"in memory, rejecting the zone transfer",
4773 			(unsigned)startserial, (unsigned)transfer_serial,
4774 			(unsigned)xfr_serial);
4775 		return 0;
4776 	}
4777 	/* everything OK in second SOA serial */
4778 	return 1;
4779 }
4780 
4781 /** apply IXFR to zone in memory. z is locked. false on failure(mallocfail) */
4782 static int
4783 apply_ixfr(struct auth_xfer* xfr, struct auth_zone* z,
4784 	struct sldns_buffer* scratch_buffer)
4785 {
4786 	struct auth_chunk* rr_chunk;
4787 	int rr_num;
4788 	size_t rr_pos;
4789 	uint8_t* rr_dname, *rr_rdata;
4790 	uint16_t rr_type, rr_class, rr_rdlen;
4791 	uint32_t rr_ttl;
4792 	size_t rr_nextpos;
4793 	int have_transfer_serial = 0;
4794 	uint32_t transfer_serial = 0;
4795 	size_t rr_counter = 0;
4796 	int delmode = 0;
4797 	int softfail = 0;
4798 
4799 	/* start RR iterator over chunklist of packets */
4800 	chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4801 	while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4802 		if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4803 			&rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4804 			&rr_rdata, &rr_nextpos)) {
4805 			/* failed to parse RR */
4806 			return 0;
4807 		}
4808 		if(verbosity>=7) log_rrlist_position("apply ixfr",
4809 			rr_chunk, rr_dname, rr_type, rr_counter);
4810 		/* twiddle add/del mode and check for start and end */
4811 		if(rr_counter == 0 && rr_type != LDNS_RR_TYPE_SOA)
4812 			return 0;
4813 		if(rr_counter == 1 && rr_type != LDNS_RR_TYPE_SOA) {
4814 			/* this is an AXFR returned from the IXFR master */
4815 			/* but that should already have been detected, by
4816 			 * on_ixfr_is_axfr */
4817 			return 0;
4818 		}
4819 		if(rr_type == LDNS_RR_TYPE_SOA) {
4820 			uint32_t serial;
4821 			if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4822 			serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4823 			if(have_transfer_serial == 0) {
4824 				have_transfer_serial = 1;
4825 				transfer_serial = serial;
4826 				delmode = 1; /* gets negated below */
4827 				/* check second RR before going any further */
4828 				if(!ixfr_start_serial(rr_chunk, rr_num, rr_pos,
4829 					rr_dname, rr_type, rr_class, rr_ttl,
4830 					rr_rdlen, rr_rdata, rr_nextpos,
4831 					transfer_serial, xfr->serial)) {
4832 					return 0;
4833 				}
4834 			} else if(transfer_serial == serial) {
4835 				have_transfer_serial++;
4836 				if(rr_counter == 1) {
4837 					/* empty AXFR, with SOA; SOA; */
4838 					/* should have been detected by
4839 					 * on_ixfr_is_axfr */
4840 					return 0;
4841 				}
4842 				if(have_transfer_serial == 3) {
4843 					/* see serial three times for end */
4844 					/* eg. IXFR:
4845 					 *  SOA 3 start
4846 					 *  SOA 1 second RR, followed by del
4847 					 *  SOA 2 followed by add
4848 					 *  SOA 2 followed by del
4849 					 *  SOA 3 followed by add
4850 					 *  SOA 3 end */
4851 					/* ended by SOA record */
4852 					xfr->serial = transfer_serial;
4853 					break;
4854 				}
4855 			}
4856 			/* twiddle add/del mode */
4857 			/* switch from delete part to add part and back again
4858 			 * just before the soa, it gets deleted and added too
4859 			 * this means we switch to delete mode for the final
4860 			 * SOA(so skip that one) */
4861 			delmode = !delmode;
4862 		}
4863 		/* process this RR */
4864 		/* if the RR is deleted twice or added twice, then we
4865 		 * softfail, and continue with the rest of the IXFR, so
4866 		 * that we serve something fairly nice during the refetch */
4867 		if(verbosity>=7) log_rrlist_position((delmode?"del":"add"),
4868 			rr_chunk, rr_dname, rr_type, rr_counter);
4869 		if(delmode) {
4870 			/* delete this RR */
4871 			int nonexist = 0;
4872 			if(!az_remove_rr_decompress(z, rr_chunk->data,
4873 				rr_chunk->len, scratch_buffer, rr_dname,
4874 				rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4875 				&nonexist)) {
4876 				/* failed, malloc error or so */
4877 				return 0;
4878 			}
4879 			if(nonexist) {
4880 				/* it was removal of a nonexisting RR */
4881 				if(verbosity>=4) log_rrlist_position(
4882 					"IXFR error nonexistent RR",
4883 					rr_chunk, rr_dname, rr_type, rr_counter);
4884 				softfail = 1;
4885 			}
4886 		} else if(rr_counter != 0) {
4887 			/* skip first SOA RR for addition, it is added in
4888 			 * the addition part near the end of the ixfr, when
4889 			 * that serial is seen the second time. */
4890 			int duplicate = 0;
4891 			/* add this RR */
4892 			if(!az_insert_rr_decompress(z, rr_chunk->data,
4893 				rr_chunk->len, scratch_buffer, rr_dname,
4894 				rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4895 				&duplicate)) {
4896 				/* failed, malloc error or so */
4897 				return 0;
4898 			}
4899 			if(duplicate) {
4900 				/* it was a duplicate */
4901 				if(verbosity>=4) log_rrlist_position(
4902 					"IXFR error duplicate RR",
4903 					rr_chunk, rr_dname, rr_type, rr_counter);
4904 				softfail = 1;
4905 			}
4906 		}
4907 
4908 		rr_counter++;
4909 		chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4910 	}
4911 	if(softfail) {
4912 		verbose(VERB_ALGO, "IXFR did not apply cleanly, fetching full zone");
4913 		return 0;
4914 	}
4915 	return 1;
4916 }
4917 
4918 /** apply AXFR to zone in memory. z is locked. false on failure(mallocfail) */
4919 static int
4920 apply_axfr(struct auth_xfer* xfr, struct auth_zone* z,
4921 	struct sldns_buffer* scratch_buffer)
4922 {
4923 	struct auth_chunk* rr_chunk;
4924 	int rr_num;
4925 	size_t rr_pos;
4926 	uint8_t* rr_dname, *rr_rdata;
4927 	uint16_t rr_type, rr_class, rr_rdlen;
4928 	uint32_t rr_ttl;
4929 	uint32_t serial = 0;
4930 	size_t rr_nextpos;
4931 	size_t rr_counter = 0;
4932 	int have_end_soa = 0;
4933 
4934 	/* clear the data tree */
4935 	traverse_postorder(&z->data, auth_data_del, NULL);
4936 	rbtree_init(&z->data, &auth_data_cmp);
4937 	/* clear the RPZ policies */
4938 	if(z->rpz)
4939 		rpz_clear(z->rpz);
4940 
4941 	xfr->have_zone = 0;
4942 	xfr->serial = 0;
4943 
4944 	/* insert all RRs in to the zone */
4945 	/* insert the SOA only once, skip the last one */
4946 	/* start RR iterator over chunklist of packets */
4947 	chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4948 	while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4949 		if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4950 			&rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4951 			&rr_rdata, &rr_nextpos)) {
4952 			/* failed to parse RR */
4953 			return 0;
4954 		}
4955 		if(verbosity>=7) log_rrlist_position("apply_axfr",
4956 			rr_chunk, rr_dname, rr_type, rr_counter);
4957 		if(rr_type == LDNS_RR_TYPE_SOA) {
4958 			if(rr_counter != 0) {
4959 				/* end of the axfr */
4960 				have_end_soa = 1;
4961 				break;
4962 			}
4963 			if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4964 			serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4965 		}
4966 
4967 		/* add this RR */
4968 		if(!az_insert_rr_decompress(z, rr_chunk->data, rr_chunk->len,
4969 			scratch_buffer, rr_dname, rr_type, rr_class, rr_ttl,
4970 			rr_rdata, rr_rdlen, NULL)) {
4971 			/* failed, malloc error or so */
4972 			return 0;
4973 		}
4974 
4975 		rr_counter++;
4976 		chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4977 	}
4978 	if(!have_end_soa) {
4979 		log_err("no end SOA record for AXFR");
4980 		return 0;
4981 	}
4982 
4983 	xfr->serial = serial;
4984 	xfr->have_zone = 1;
4985 	return 1;
4986 }
4987 
4988 /** apply HTTP to zone in memory. z is locked. false on failure(mallocfail) */
4989 static int
4990 apply_http(struct auth_xfer* xfr, struct auth_zone* z,
4991 	struct sldns_buffer* scratch_buffer)
4992 {
4993 	/* parse data in chunks */
4994 	/* parse RR's and read into memory. ignore $INCLUDE from the
4995 	 * downloaded file*/
4996 	struct sldns_file_parse_state pstate;
4997 	struct auth_chunk* chunk;
4998 	size_t chunk_pos;
4999 	memset(&pstate, 0, sizeof(pstate));
5000 	pstate.default_ttl = 3600;
5001 	if(xfr->namelen < sizeof(pstate.origin)) {
5002 		pstate.origin_len = xfr->namelen;
5003 		memmove(pstate.origin, xfr->name, xfr->namelen);
5004 	}
5005 
5006 	if(verbosity >= VERB_ALGO)
5007 		verbose(VERB_ALGO, "http download %s of size %d",
5008 		xfr->task_transfer->master->file,
5009 		(int)chunklist_sum(xfr->task_transfer->chunks_first));
5010 	if(xfr->task_transfer->chunks_first && verbosity >= VERB_ALGO) {
5011 		char preview[1024];
5012 		if(xfr->task_transfer->chunks_first->len+1 > sizeof(preview)) {
5013 			memmove(preview, xfr->task_transfer->chunks_first->data,
5014 				sizeof(preview)-1);
5015 			preview[sizeof(preview)-1]=0;
5016 		} else {
5017 			memmove(preview, xfr->task_transfer->chunks_first->data,
5018 				xfr->task_transfer->chunks_first->len);
5019 			preview[xfr->task_transfer->chunks_first->len]=0;
5020 		}
5021 		log_info("auth zone http downloaded content preview: %s",
5022 			preview);
5023 	}
5024 
5025 	/* perhaps a little syntax check before we try to apply the data? */
5026 	if(!http_zonefile_syntax_check(xfr, scratch_buffer)) {
5027 		log_err("http download %s/%s does not contain a zonefile, "
5028 			"but got '%s'", xfr->task_transfer->master->host,
5029 			xfr->task_transfer->master->file,
5030 			sldns_buffer_begin(scratch_buffer));
5031 		return 0;
5032 	}
5033 
5034 	/* clear the data tree */
5035 	traverse_postorder(&z->data, auth_data_del, NULL);
5036 	rbtree_init(&z->data, &auth_data_cmp);
5037 	/* clear the RPZ policies */
5038 	if(z->rpz)
5039 		rpz_clear(z->rpz);
5040 
5041 	xfr->have_zone = 0;
5042 	xfr->serial = 0;
5043 
5044 	chunk = xfr->task_transfer->chunks_first;
5045 	chunk_pos = 0;
5046 	pstate.lineno = 0;
5047 	while(chunkline_get_line_collated(&chunk, &chunk_pos, scratch_buffer)) {
5048 		/* process this line */
5049 		pstate.lineno++;
5050 		chunkline_newline_removal(scratch_buffer);
5051 		if(chunkline_is_comment_line_or_empty(scratch_buffer)) {
5052 			continue;
5053 		}
5054 		/* parse line and add RR */
5055 		if(http_parse_origin(scratch_buffer, &pstate)) {
5056 			continue; /* $ORIGIN has been handled */
5057 		}
5058 		if(http_parse_ttl(scratch_buffer, &pstate)) {
5059 			continue; /* $TTL has been handled */
5060 		}
5061 		if(!http_parse_add_rr(xfr, z, scratch_buffer, &pstate)) {
5062 			verbose(VERB_ALGO, "error parsing line [%s:%d] %s",
5063 				xfr->task_transfer->master->file,
5064 				pstate.lineno,
5065 				sldns_buffer_begin(scratch_buffer));
5066 			return 0;
5067 		}
5068 	}
5069 	return 1;
5070 }
5071 
5072 /** write http chunks to zonefile to create downloaded file */
5073 static int
5074 auth_zone_write_chunks(struct auth_xfer* xfr, const char* fname)
5075 {
5076 	FILE* out;
5077 	struct auth_chunk* p;
5078 	out = fopen(fname, "w");
5079 	if(!out) {
5080 		log_err("could not open %s: %s", fname, strerror(errno));
5081 		return 0;
5082 	}
5083 	for(p = xfr->task_transfer->chunks_first; p ; p = p->next) {
5084 		if(!write_out(out, (char*)p->data, p->len)) {
5085 			log_err("could not write http download to %s", fname);
5086 			fclose(out);
5087 			return 0;
5088 		}
5089 	}
5090 	fclose(out);
5091 	return 1;
5092 }
5093 
5094 /** write to zonefile after zone has been updated */
5095 static void
5096 xfr_write_after_update(struct auth_xfer* xfr, struct module_env* env)
5097 {
5098 	struct config_file* cfg = env->cfg;
5099 	struct auth_zone* z;
5100 	char tmpfile[1024];
5101 	char* zfilename;
5102 	lock_basic_unlock(&xfr->lock);
5103 
5104 	/* get lock again, so it is a readlock and concurrently queries
5105 	 * can be answered */
5106 	lock_rw_rdlock(&env->auth_zones->lock);
5107 	z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
5108 		xfr->dclass);
5109 	if(!z) {
5110 		lock_rw_unlock(&env->auth_zones->lock);
5111 		/* the zone is gone, ignore xfr results */
5112 		lock_basic_lock(&xfr->lock);
5113 		return;
5114 	}
5115 	lock_rw_rdlock(&z->lock);
5116 	lock_basic_lock(&xfr->lock);
5117 	lock_rw_unlock(&env->auth_zones->lock);
5118 
5119 	if(z->zonefile == NULL || z->zonefile[0] == 0) {
5120 		lock_rw_unlock(&z->lock);
5121 		/* no write needed, no zonefile set */
5122 		return;
5123 	}
5124 	zfilename = z->zonefile;
5125 	if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
5126 		cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
5127 		zfilename += strlen(cfg->chrootdir);
5128 	if(verbosity >= VERB_ALGO) {
5129 		char nm[255+1];
5130 		dname_str(z->name, nm);
5131 		verbose(VERB_ALGO, "write zonefile %s for %s", zfilename, nm);
5132 	}
5133 
5134 	/* write to tempfile first */
5135 	if((size_t)strlen(zfilename) + 16 > sizeof(tmpfile)) {
5136 		verbose(VERB_ALGO, "tmpfilename too long, cannot update "
5137 			" zonefile %s", zfilename);
5138 		lock_rw_unlock(&z->lock);
5139 		return;
5140 	}
5141 	snprintf(tmpfile, sizeof(tmpfile), "%s.tmp%u", zfilename,
5142 		(unsigned)getpid());
5143 	if(xfr->task_transfer->master->http) {
5144 		/* use the stored chunk list to write them */
5145 		if(!auth_zone_write_chunks(xfr, tmpfile)) {
5146 			unlink(tmpfile);
5147 			lock_rw_unlock(&z->lock);
5148 			return;
5149 		}
5150 	} else if(!auth_zone_write_file(z, tmpfile)) {
5151 		unlink(tmpfile);
5152 		lock_rw_unlock(&z->lock);
5153 		return;
5154 	}
5155 #ifdef UB_ON_WINDOWS
5156 	(void)unlink(zfilename); /* windows does not replace file with rename() */
5157 #endif
5158 	if(rename(tmpfile, zfilename) < 0) {
5159 		log_err("could not rename(%s, %s): %s", tmpfile, zfilename,
5160 			strerror(errno));
5161 		unlink(tmpfile);
5162 		lock_rw_unlock(&z->lock);
5163 		return;
5164 	}
5165 	lock_rw_unlock(&z->lock);
5166 }
5167 
5168 /** reacquire locks and structures. Starts with no locks, ends
5169  * with xfr and z locks, if fail, no z lock */
5170 static int xfr_process_reacquire_locks(struct auth_xfer* xfr,
5171 	struct module_env* env, struct auth_zone** z)
5172 {
5173 	/* release xfr lock, then, while holding az->lock grab both
5174 	 * z->lock and xfr->lock */
5175 	lock_rw_rdlock(&env->auth_zones->lock);
5176 	*z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
5177 		xfr->dclass);
5178 	if(!*z) {
5179 		lock_rw_unlock(&env->auth_zones->lock);
5180 		lock_basic_lock(&xfr->lock);
5181 		*z = NULL;
5182 		return 0;
5183 	}
5184 	lock_rw_wrlock(&(*z)->lock);
5185 	lock_basic_lock(&xfr->lock);
5186 	lock_rw_unlock(&env->auth_zones->lock);
5187 	return 1;
5188 }
5189 
5190 /** process chunk list and update zone in memory,
5191  * return false if it did not work */
5192 static int
5193 xfr_process_chunk_list(struct auth_xfer* xfr, struct module_env* env,
5194 	int* ixfr_fail)
5195 {
5196 	struct auth_zone* z;
5197 
5198 	/* obtain locks and structures */
5199 	lock_basic_unlock(&xfr->lock);
5200 	if(!xfr_process_reacquire_locks(xfr, env, &z)) {
5201 		/* the zone is gone, ignore xfr results */
5202 		return 0;
5203 	}
5204 	/* holding xfr and z locks */
5205 
5206 	/* apply data */
5207 	if(xfr->task_transfer->master->http) {
5208 		if(!apply_http(xfr, z, env->scratch_buffer)) {
5209 			lock_rw_unlock(&z->lock);
5210 			verbose(VERB_ALGO, "http from %s: could not store data",
5211 				xfr->task_transfer->master->host);
5212 			return 0;
5213 		}
5214 	} else if(xfr->task_transfer->on_ixfr &&
5215 		!xfr->task_transfer->on_ixfr_is_axfr) {
5216 		if(!apply_ixfr(xfr, z, env->scratch_buffer)) {
5217 			lock_rw_unlock(&z->lock);
5218 			verbose(VERB_ALGO, "xfr from %s: could not store IXFR"
5219 				" data", xfr->task_transfer->master->host);
5220 			*ixfr_fail = 1;
5221 			return 0;
5222 		}
5223 	} else {
5224 		if(!apply_axfr(xfr, z, env->scratch_buffer)) {
5225 			lock_rw_unlock(&z->lock);
5226 			verbose(VERB_ALGO, "xfr from %s: could not store AXFR"
5227 				" data", xfr->task_transfer->master->host);
5228 			return 0;
5229 		}
5230 	}
5231 	xfr->zone_expired = 0;
5232 	z->zone_expired = 0;
5233 	if(!xfr_find_soa(z, xfr)) {
5234 		lock_rw_unlock(&z->lock);
5235 		verbose(VERB_ALGO, "xfr from %s: no SOA in zone after update"
5236 			" (or malformed RR)", xfr->task_transfer->master->host);
5237 		return 0;
5238 	}
5239 
5240 	/* release xfr lock while verifying zonemd because it may have
5241 	 * to spawn lookups in the state machines */
5242 	lock_basic_unlock(&xfr->lock);
5243 	/* holding z lock */
5244 	auth_zone_verify_zonemd(z, env, &env->mesh->mods, NULL, 0, 0);
5245 	if(z->zone_expired) {
5246 		char zname[256];
5247 		/* ZONEMD must have failed */
5248 		/* reacquire locks, so we hold xfr lock on exit of routine,
5249 		 * and both xfr and z again after releasing xfr for potential
5250 		 * state machine mesh callbacks */
5251 		lock_rw_unlock(&z->lock);
5252 		if(!xfr_process_reacquire_locks(xfr, env, &z))
5253 			return 0;
5254 		dname_str(xfr->name, zname);
5255 		verbose(VERB_ALGO, "xfr from %s: ZONEMD failed for %s, transfer is failed", xfr->task_transfer->master->host, zname);
5256 		xfr->zone_expired = 1;
5257 		lock_rw_unlock(&z->lock);
5258 		return 0;
5259 	}
5260 	/* reacquire locks, so we hold xfr lock on exit of routine,
5261 	 * and both xfr and z again after releasing xfr for potential
5262 	 * state machine mesh callbacks */
5263 	lock_rw_unlock(&z->lock);
5264 	if(!xfr_process_reacquire_locks(xfr, env, &z))
5265 		return 0;
5266 	/* holding xfr and z locks */
5267 
5268 	if(xfr->have_zone)
5269 		xfr->lease_time = *env->now;
5270 
5271 	if(z->rpz)
5272 		rpz_finish_config(z->rpz);
5273 
5274 	/* unlock */
5275 	lock_rw_unlock(&z->lock);
5276 
5277 	if(verbosity >= VERB_QUERY && xfr->have_zone) {
5278 		char zname[256];
5279 		dname_str(xfr->name, zname);
5280 		verbose(VERB_QUERY, "auth zone %s updated to serial %u", zname,
5281 			(unsigned)xfr->serial);
5282 	}
5283 	/* see if we need to write to a zonefile */
5284 	xfr_write_after_update(xfr, env);
5285 	return 1;
5286 }
5287 
5288 /** disown task_transfer.  caller must hold xfr.lock */
5289 static void
5290 xfr_transfer_disown(struct auth_xfer* xfr)
5291 {
5292 	/* remove timer (from this worker's event base) */
5293 	comm_timer_delete(xfr->task_transfer->timer);
5294 	xfr->task_transfer->timer = NULL;
5295 	/* remove the commpoint */
5296 	comm_point_delete(xfr->task_transfer->cp);
5297 	xfr->task_transfer->cp = NULL;
5298 	/* we don't own this item anymore */
5299 	xfr->task_transfer->worker = NULL;
5300 	xfr->task_transfer->env = NULL;
5301 }
5302 
5303 /** lookup a host name for its addresses, if needed */
5304 static int
5305 xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
5306 {
5307 	struct sockaddr_storage addr;
5308 	socklen_t addrlen = 0;
5309 	struct auth_master* master = xfr->task_transfer->lookup_target;
5310 	struct query_info qinfo;
5311 	uint16_t qflags = BIT_RD;
5312 	uint8_t dname[LDNS_MAX_DOMAINLEN+1];
5313 	struct edns_data edns;
5314 	sldns_buffer* buf = env->scratch_buffer;
5315 	if(!master) return 0;
5316 	if(extstrtoaddr(master->host, &addr, &addrlen)) {
5317 		/* not needed, host is in IP addr format */
5318 		return 0;
5319 	}
5320 	if(master->allow_notify)
5321 		return 0; /* allow-notifies are not transferred from, no
5322 		lookup is needed */
5323 
5324 	/* use mesh_new_callback to probe for non-addr hosts,
5325 	 * and then wait for them to be looked up (in cache, or query) */
5326 	qinfo.qname_len = sizeof(dname);
5327 	if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
5328 		!= 0) {
5329 		log_err("cannot parse host name of master %s", master->host);
5330 		return 0;
5331 	}
5332 	qinfo.qname = dname;
5333 	qinfo.qclass = xfr->dclass;
5334 	qinfo.qtype = LDNS_RR_TYPE_A;
5335 	if(xfr->task_transfer->lookup_aaaa)
5336 		qinfo.qtype = LDNS_RR_TYPE_AAAA;
5337 	qinfo.local_alias = NULL;
5338 	if(verbosity >= VERB_ALGO) {
5339 		char buf1[512];
5340 		char buf2[LDNS_MAX_DOMAINLEN+1];
5341 		dname_str(xfr->name, buf2);
5342 		snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
5343 			" for task_transfer", buf2);
5344 		log_query_info(VERB_ALGO, buf1, &qinfo);
5345 	}
5346 	edns.edns_present = 1;
5347 	edns.ext_rcode = 0;
5348 	edns.edns_version = 0;
5349 	edns.bits = EDNS_DO;
5350 	edns.opt_list = NULL;
5351 	edns.padding_block_size = 0;
5352 	if(sldns_buffer_capacity(buf) < 65535)
5353 		edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
5354 	else	edns.udp_size = 65535;
5355 
5356 	/* unlock xfr during mesh_new_callback() because the callback can be
5357 	 * called straight away */
5358 	lock_basic_unlock(&xfr->lock);
5359 	if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
5360 		&auth_xfer_transfer_lookup_callback, xfr)) {
5361 		lock_basic_lock(&xfr->lock);
5362 		log_err("out of memory lookup up master %s", master->host);
5363 		return 0;
5364 	}
5365 	lock_basic_lock(&xfr->lock);
5366 	return 1;
5367 }
5368 
5369 /** initiate TCP to the target and fetch zone.
5370  * returns true if that was successfully started, and timeout setup. */
5371 static int
5372 xfr_transfer_init_fetch(struct auth_xfer* xfr, struct module_env* env)
5373 {
5374 	struct sockaddr_storage addr;
5375 	socklen_t addrlen = 0;
5376 	struct auth_master* master = xfr->task_transfer->master;
5377 	char *auth_name = NULL;
5378 	struct timeval t;
5379 	int timeout;
5380 	if(!master) return 0;
5381 	if(master->allow_notify) return 0; /* only for notify */
5382 
5383 	/* get master addr */
5384 	if(xfr->task_transfer->scan_addr) {
5385 		addrlen = xfr->task_transfer->scan_addr->addrlen;
5386 		memmove(&addr, &xfr->task_transfer->scan_addr->addr, addrlen);
5387 	} else {
5388 		if(!authextstrtoaddr(master->host, &addr, &addrlen, &auth_name)) {
5389 			/* the ones that are not in addr format are supposed
5390 			 * to be looked up.  The lookup has failed however,
5391 			 * so skip them */
5392 			char zname[255+1];
5393 			dname_str(xfr->name, zname);
5394 			log_err("%s: failed lookup, cannot transfer from master %s",
5395 				zname, master->host);
5396 			return 0;
5397 		}
5398 	}
5399 
5400 	/* remove previous TCP connection (if any) */
5401 	if(xfr->task_transfer->cp) {
5402 		comm_point_delete(xfr->task_transfer->cp);
5403 		xfr->task_transfer->cp = NULL;
5404 	}
5405 	if(!xfr->task_transfer->timer) {
5406 		xfr->task_transfer->timer = comm_timer_create(env->worker_base,
5407 			auth_xfer_transfer_timer_callback, xfr);
5408 		if(!xfr->task_transfer->timer) {
5409 			log_err("malloc failure");
5410 			return 0;
5411 		}
5412 	}
5413 	timeout = AUTH_TRANSFER_TIMEOUT;
5414 #ifndef S_SPLINT_S
5415         t.tv_sec = timeout/1000;
5416         t.tv_usec = (timeout%1000)*1000;
5417 #endif
5418 
5419 	if(master->http) {
5420 		/* perform http fetch */
5421 		/* store http port number into sockaddr,
5422 		 * unless someone used unbound's host@port notation */
5423 		xfr->task_transfer->on_ixfr = 0;
5424 		if(strchr(master->host, '@') == NULL)
5425 			sockaddr_store_port(&addr, addrlen, master->port);
5426 		xfr->task_transfer->cp = outnet_comm_point_for_http(
5427 			env->outnet, auth_xfer_transfer_http_callback, xfr,
5428 			&addr, addrlen, -1, master->ssl, master->host,
5429 			master->file, env->cfg);
5430 		if(!xfr->task_transfer->cp) {
5431 			char zname[255+1], as[256];
5432 			dname_str(xfr->name, zname);
5433 			addr_to_str(&addr, addrlen, as, sizeof(as));
5434 			verbose(VERB_ALGO, "cannot create http cp "
5435 				"connection for %s to %s", zname, as);
5436 			return 0;
5437 		}
5438 		comm_timer_set(xfr->task_transfer->timer, &t);
5439 		if(verbosity >= VERB_ALGO) {
5440 			char zname[255+1], as[256];
5441 			dname_str(xfr->name, zname);
5442 			addr_to_str(&addr, addrlen, as, sizeof(as));
5443 			verbose(VERB_ALGO, "auth zone %s transfer next HTTP fetch from %s started", zname, as);
5444 		}
5445 		return 1;
5446 	}
5447 
5448 	/* perform AXFR/IXFR */
5449 	/* set the packet to be written */
5450 	/* create new ID */
5451 	xfr->task_transfer->id = GET_RANDOM_ID(env->rnd);
5452 	xfr_create_ixfr_packet(xfr, env->scratch_buffer,
5453 		xfr->task_transfer->id, master);
5454 
5455 	/* connect on fd */
5456 	xfr->task_transfer->cp = outnet_comm_point_for_tcp(env->outnet,
5457 		auth_xfer_transfer_tcp_callback, xfr, &addr, addrlen,
5458 		env->scratch_buffer, -1,
5459 		auth_name != NULL, auth_name);
5460 	if(!xfr->task_transfer->cp) {
5461 		char zname[255+1], as[256];
5462  		dname_str(xfr->name, zname);
5463 		addr_to_str(&addr, addrlen, as, sizeof(as));
5464 		verbose(VERB_ALGO, "cannot create tcp cp connection for "
5465 			"xfr %s to %s", zname, as);
5466 		return 0;
5467 	}
5468 	comm_timer_set(xfr->task_transfer->timer, &t);
5469 	if(verbosity >= VERB_ALGO) {
5470 		char zname[255+1], as[256];
5471  		dname_str(xfr->name, zname);
5472 		addr_to_str(&addr, addrlen, as, sizeof(as));
5473 		verbose(VERB_ALGO, "auth zone %s transfer next %s fetch from %s started", zname,
5474 			(xfr->task_transfer->on_ixfr?"IXFR":"AXFR"), as);
5475 	}
5476 	return 1;
5477 }
5478 
5479 /** perform next lookup, next transfer TCP, or end and resume wait time task */
5480 static void
5481 xfr_transfer_nexttarget_or_end(struct auth_xfer* xfr, struct module_env* env)
5482 {
5483 	log_assert(xfr->task_transfer->worker == env->worker);
5484 
5485 	/* are we performing lookups? */
5486 	while(xfr->task_transfer->lookup_target) {
5487 		if(xfr_transfer_lookup_host(xfr, env)) {
5488 			/* wait for lookup to finish,
5489 			 * note that the hostname may be in unbound's cache
5490 			 * and we may then get an instant cache response,
5491 			 * and that calls the callback just like a full
5492 			 * lookup and lookup failures also call callback */
5493 			if(verbosity >= VERB_ALGO) {
5494 				char zname[255+1];
5495 				dname_str(xfr->name, zname);
5496 				verbose(VERB_ALGO, "auth zone %s transfer next target lookup", zname);
5497 			}
5498 			lock_basic_unlock(&xfr->lock);
5499 			return;
5500 		}
5501 		xfr_transfer_move_to_next_lookup(xfr, env);
5502 	}
5503 
5504 	/* initiate TCP and fetch the zone from the master */
5505 	/* and set timeout on it */
5506 	while(!xfr_transfer_end_of_list(xfr)) {
5507 		xfr->task_transfer->master = xfr_transfer_current_master(xfr);
5508 		if(xfr_transfer_init_fetch(xfr, env)) {
5509 			/* successfully started, wait for callback */
5510 			lock_basic_unlock(&xfr->lock);
5511 			return;
5512 		}
5513 		/* failed to fetch, next master */
5514 		xfr_transfer_nextmaster(xfr);
5515 	}
5516 	if(verbosity >= VERB_ALGO) {
5517 		char zname[255+1];
5518 		dname_str(xfr->name, zname);
5519 		verbose(VERB_ALGO, "auth zone %s transfer failed, wait", zname);
5520 	}
5521 
5522 	/* we failed to fetch the zone, move to wait task
5523 	 * use the shorter retry timeout */
5524 	xfr_transfer_disown(xfr);
5525 
5526 	/* pick up the nextprobe task and wait */
5527 	if(xfr->task_nextprobe->worker == NULL)
5528 		xfr_set_timeout(xfr, env, 1, 0);
5529 	lock_basic_unlock(&xfr->lock);
5530 }
5531 
5532 /** add addrs from A or AAAA rrset to the master */
5533 static void
5534 xfr_master_add_addrs(struct auth_master* m, struct ub_packed_rrset_key* rrset,
5535 	uint16_t rrtype)
5536 {
5537 	size_t i;
5538 	struct packed_rrset_data* data;
5539 	if(!m || !rrset) return;
5540 	if(rrtype != LDNS_RR_TYPE_A && rrtype != LDNS_RR_TYPE_AAAA)
5541 		return;
5542 	data = (struct packed_rrset_data*)rrset->entry.data;
5543 	for(i=0; i<data->count; i++) {
5544 		struct auth_addr* a;
5545 		size_t len = data->rr_len[i] - 2;
5546 		uint8_t* rdata = data->rr_data[i]+2;
5547 		if(rrtype == LDNS_RR_TYPE_A && len != INET_SIZE)
5548 			continue; /* wrong length for A */
5549 		if(rrtype == LDNS_RR_TYPE_AAAA && len != INET6_SIZE)
5550 			continue; /* wrong length for AAAA */
5551 
5552 		/* add and alloc it */
5553 		a = (struct auth_addr*)calloc(1, sizeof(*a));
5554 		if(!a) {
5555 			log_err("out of memory");
5556 			return;
5557 		}
5558 		if(rrtype == LDNS_RR_TYPE_A) {
5559 			struct sockaddr_in* sa;
5560 			a->addrlen = (socklen_t)sizeof(*sa);
5561 			sa = (struct sockaddr_in*)&a->addr;
5562 			sa->sin_family = AF_INET;
5563 			sa->sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5564 			memmove(&sa->sin_addr, rdata, INET_SIZE);
5565 		} else {
5566 			struct sockaddr_in6* sa;
5567 			a->addrlen = (socklen_t)sizeof(*sa);
5568 			sa = (struct sockaddr_in6*)&a->addr;
5569 			sa->sin6_family = AF_INET6;
5570 			sa->sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5571 			memmove(&sa->sin6_addr, rdata, INET6_SIZE);
5572 		}
5573 		if(verbosity >= VERB_ALGO) {
5574 			char s[64];
5575 			addr_to_str(&a->addr, a->addrlen, s, sizeof(s));
5576 			verbose(VERB_ALGO, "auth host %s lookup %s",
5577 				m->host, s);
5578 		}
5579 		/* append to list */
5580 		a->next = m->list;
5581 		m->list = a;
5582 	}
5583 }
5584 
5585 /** callback for task_transfer lookup of host name, of A or AAAA */
5586 void auth_xfer_transfer_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
5587 	enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
5588 	int ATTR_UNUSED(was_ratelimited))
5589 {
5590 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
5591 	struct module_env* env;
5592 	log_assert(xfr->task_transfer);
5593 	lock_basic_lock(&xfr->lock);
5594 	env = xfr->task_transfer->env;
5595 	if(!env || env->outnet->want_to_quit) {
5596 		lock_basic_unlock(&xfr->lock);
5597 		return; /* stop on quit */
5598 	}
5599 
5600 	/* process result */
5601 	if(rcode == LDNS_RCODE_NOERROR) {
5602 		uint16_t wanted_qtype = LDNS_RR_TYPE_A;
5603 		struct regional* temp = env->scratch;
5604 		struct query_info rq;
5605 		struct reply_info* rep;
5606 		if(xfr->task_transfer->lookup_aaaa)
5607 			wanted_qtype = LDNS_RR_TYPE_AAAA;
5608 		memset(&rq, 0, sizeof(rq));
5609 		rep = parse_reply_in_temp_region(buf, temp, &rq);
5610 		if(rep && rq.qtype == wanted_qtype &&
5611 			FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
5612 			/* parsed successfully */
5613 			struct ub_packed_rrset_key* answer =
5614 				reply_find_answer_rrset(&rq, rep);
5615 			if(answer) {
5616 				xfr_master_add_addrs(xfr->task_transfer->
5617 					lookup_target, answer, wanted_qtype);
5618 			} else {
5619 				if(verbosity >= VERB_ALGO) {
5620 					char zname[255+1];
5621 					dname_str(xfr->name, zname);
5622 					verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup has nodata", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5623 				}
5624 			}
5625 		} else {
5626 			if(verbosity >= VERB_ALGO) {
5627 				char zname[255+1];
5628 				dname_str(xfr->name, zname);
5629 				verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup has no answer", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5630 			}
5631 		}
5632 		regional_free_all(temp);
5633 	} else {
5634 		if(verbosity >= VERB_ALGO) {
5635 			char zname[255+1];
5636 			dname_str(xfr->name, zname);
5637 			verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup failed", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5638 		}
5639 	}
5640 	if(xfr->task_transfer->lookup_target->list &&
5641 		xfr->task_transfer->lookup_target == xfr_transfer_current_master(xfr))
5642 		xfr->task_transfer->scan_addr = xfr->task_transfer->lookup_target->list;
5643 
5644 	/* move to lookup AAAA after A lookup, move to next hostname lookup,
5645 	 * or move to fetch the zone, or, if nothing to do, end task_transfer */
5646 	xfr_transfer_move_to_next_lookup(xfr, env);
5647 	xfr_transfer_nexttarget_or_end(xfr, env);
5648 }
5649 
5650 /** check if xfer (AXFR or IXFR) packet is OK.
5651  * return false if we lost connection (SERVFAIL, or unreadable).
5652  * return false if we need to move from IXFR to AXFR, with gonextonfail
5653  * 	set to false, so the same master is tried again, but with AXFR.
5654  * return true if fine to link into data.
5655  * return true with transferdone=true when the transfer has ended.
5656  */
5657 static int
5658 check_xfer_packet(sldns_buffer* pkt, struct auth_xfer* xfr,
5659 	int* gonextonfail, int* transferdone)
5660 {
5661 	uint8_t* wire = sldns_buffer_begin(pkt);
5662 	int i;
5663 	if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE) {
5664 		verbose(VERB_ALGO, "xfr to %s failed, packet too small",
5665 			xfr->task_transfer->master->host);
5666 		return 0;
5667 	}
5668 	if(!LDNS_QR_WIRE(wire)) {
5669 		verbose(VERB_ALGO, "xfr to %s failed, packet has no QR flag",
5670 			xfr->task_transfer->master->host);
5671 		return 0;
5672 	}
5673 	if(LDNS_TC_WIRE(wire)) {
5674 		verbose(VERB_ALGO, "xfr to %s failed, packet has TC flag",
5675 			xfr->task_transfer->master->host);
5676 		return 0;
5677 	}
5678 	/* check ID */
5679 	if(LDNS_ID_WIRE(wire) != xfr->task_transfer->id) {
5680 		verbose(VERB_ALGO, "xfr to %s failed, packet wrong ID",
5681 			xfr->task_transfer->master->host);
5682 		return 0;
5683 	}
5684 	if(LDNS_RCODE_WIRE(wire) != LDNS_RCODE_NOERROR) {
5685 		char rcode[32];
5686 		sldns_wire2str_rcode_buf((int)LDNS_RCODE_WIRE(wire), rcode,
5687 			sizeof(rcode));
5688 		/* if we are doing IXFR, check for fallback */
5689 		if(xfr->task_transfer->on_ixfr) {
5690 			if(LDNS_RCODE_WIRE(wire) == LDNS_RCODE_NOTIMPL ||
5691 				LDNS_RCODE_WIRE(wire) == LDNS_RCODE_SERVFAIL ||
5692 				LDNS_RCODE_WIRE(wire) == LDNS_RCODE_REFUSED ||
5693 				LDNS_RCODE_WIRE(wire) == LDNS_RCODE_FORMERR) {
5694 				verbose(VERB_ALGO, "xfr to %s, fallback "
5695 					"from IXFR to AXFR (with rcode %s)",
5696 					xfr->task_transfer->master->host,
5697 					rcode);
5698 				xfr->task_transfer->ixfr_fail = 1;
5699 				*gonextonfail = 0;
5700 				return 0;
5701 			}
5702 		}
5703 		verbose(VERB_ALGO, "xfr to %s failed, packet with rcode %s",
5704 			xfr->task_transfer->master->host, rcode);
5705 		return 0;
5706 	}
5707 	if(LDNS_OPCODE_WIRE(wire) != LDNS_PACKET_QUERY) {
5708 		verbose(VERB_ALGO, "xfr to %s failed, packet with bad opcode",
5709 			xfr->task_transfer->master->host);
5710 		return 0;
5711 	}
5712 	if(LDNS_QDCOUNT(wire) > 1) {
5713 		verbose(VERB_ALGO, "xfr to %s failed, packet has qdcount %d",
5714 			xfr->task_transfer->master->host,
5715 			(int)LDNS_QDCOUNT(wire));
5716 		return 0;
5717 	}
5718 
5719 	/* check qname */
5720 	sldns_buffer_set_position(pkt, LDNS_HEADER_SIZE);
5721 	for(i=0; i<(int)LDNS_QDCOUNT(wire); i++) {
5722 		size_t pos = sldns_buffer_position(pkt);
5723 		uint16_t qtype, qclass;
5724 		if(pkt_dname_len(pkt) == 0) {
5725 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5726 				"malformed dname",
5727 				xfr->task_transfer->master->host);
5728 			return 0;
5729 		}
5730 		if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5731 			xfr->name) != 0) {
5732 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5733 				"wrong qname",
5734 				xfr->task_transfer->master->host);
5735 			return 0;
5736 		}
5737 		if(sldns_buffer_remaining(pkt) < 4) {
5738 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5739 				"truncated query RR",
5740 				xfr->task_transfer->master->host);
5741 			return 0;
5742 		}
5743 		qtype = sldns_buffer_read_u16(pkt);
5744 		qclass = sldns_buffer_read_u16(pkt);
5745 		if(qclass != xfr->dclass) {
5746 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5747 				"wrong qclass",
5748 				xfr->task_transfer->master->host);
5749 			return 0;
5750 		}
5751 		if(xfr->task_transfer->on_ixfr) {
5752 			if(qtype != LDNS_RR_TYPE_IXFR) {
5753 				verbose(VERB_ALGO, "xfr to %s failed, packet "
5754 					"with wrong qtype, expected IXFR",
5755 				xfr->task_transfer->master->host);
5756 				return 0;
5757 			}
5758 		} else {
5759 			if(qtype != LDNS_RR_TYPE_AXFR) {
5760 				verbose(VERB_ALGO, "xfr to %s failed, packet "
5761 					"with wrong qtype, expected AXFR",
5762 				xfr->task_transfer->master->host);
5763 				return 0;
5764 			}
5765 		}
5766 	}
5767 
5768 	/* check parse of RRs in packet, store first SOA serial
5769 	 * to be able to detect last SOA (with that serial) to see if done */
5770 	/* also check for IXFR 'zone up to date' reply */
5771 	for(i=0; i<(int)LDNS_ANCOUNT(wire); i++) {
5772 		size_t pos = sldns_buffer_position(pkt);
5773 		uint16_t tp, rdlen;
5774 		if(pkt_dname_len(pkt) == 0) {
5775 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5776 				"malformed dname in answer section",
5777 				xfr->task_transfer->master->host);
5778 			return 0;
5779 		}
5780 		if(sldns_buffer_remaining(pkt) < 10) {
5781 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5782 				"truncated RR",
5783 				xfr->task_transfer->master->host);
5784 			return 0;
5785 		}
5786 		tp = sldns_buffer_read_u16(pkt);
5787 		(void)sldns_buffer_read_u16(pkt); /* class */
5788 		(void)sldns_buffer_read_u32(pkt); /* ttl */
5789 		rdlen = sldns_buffer_read_u16(pkt);
5790 		if(sldns_buffer_remaining(pkt) < rdlen) {
5791 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5792 				"truncated RR rdata",
5793 				xfr->task_transfer->master->host);
5794 			return 0;
5795 		}
5796 
5797 		/* RR parses (haven't checked rdata itself), now look at
5798 		 * SOA records to see serial number */
5799 		if(xfr->task_transfer->rr_scan_num == 0 &&
5800 			tp != LDNS_RR_TYPE_SOA) {
5801 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5802 				"malformed zone transfer, no start SOA",
5803 				xfr->task_transfer->master->host);
5804 			return 0;
5805 		}
5806 		if(xfr->task_transfer->rr_scan_num == 1 &&
5807 			tp != LDNS_RR_TYPE_SOA) {
5808 			/* second RR is not a SOA record, this is not an IXFR
5809 			 * the master is replying with an AXFR */
5810 			xfr->task_transfer->on_ixfr_is_axfr = 1;
5811 		}
5812 		if(tp == LDNS_RR_TYPE_SOA) {
5813 			uint32_t serial;
5814 			if(rdlen < 22) {
5815 				verbose(VERB_ALGO, "xfr to %s failed, packet "
5816 					"with SOA with malformed rdata",
5817 					xfr->task_transfer->master->host);
5818 				return 0;
5819 			}
5820 			if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5821 				xfr->name) != 0) {
5822 				verbose(VERB_ALGO, "xfr to %s failed, packet "
5823 					"with SOA with wrong dname",
5824 					xfr->task_transfer->master->host);
5825 				return 0;
5826 			}
5827 
5828 			/* read serial number of SOA */
5829 			serial = sldns_buffer_read_u32_at(pkt,
5830 				sldns_buffer_position(pkt)+rdlen-20);
5831 
5832 			/* check for IXFR 'zone has SOA x' reply */
5833 			if(xfr->task_transfer->on_ixfr &&
5834 				xfr->task_transfer->rr_scan_num == 0 &&
5835 				LDNS_ANCOUNT(wire)==1) {
5836 				verbose(VERB_ALGO, "xfr to %s ended, "
5837 					"IXFR reply that zone has serial %u,"
5838 					" fallback from IXFR to AXFR",
5839 					xfr->task_transfer->master->host,
5840 					(unsigned)serial);
5841 				xfr->task_transfer->ixfr_fail = 1;
5842 				*gonextonfail = 0;
5843 				return 0;
5844 			}
5845 
5846 			/* if first SOA, store serial number */
5847 			if(xfr->task_transfer->got_xfr_serial == 0) {
5848 				xfr->task_transfer->got_xfr_serial = 1;
5849 				xfr->task_transfer->incoming_xfr_serial =
5850 					serial;
5851 				verbose(VERB_ALGO, "xfr %s: contains "
5852 					"SOA serial %u",
5853 					xfr->task_transfer->master->host,
5854 					(unsigned)serial);
5855 			/* see if end of AXFR */
5856 			} else if(!xfr->task_transfer->on_ixfr ||
5857 				xfr->task_transfer->on_ixfr_is_axfr) {
5858 				/* second SOA with serial is the end
5859 				 * for AXFR */
5860 				*transferdone = 1;
5861 				verbose(VERB_ALGO, "xfr %s: last AXFR packet",
5862 					xfr->task_transfer->master->host);
5863 			/* for IXFR, count SOA records with that serial */
5864 			} else if(xfr->task_transfer->incoming_xfr_serial ==
5865 				serial && xfr->task_transfer->got_xfr_serial
5866 				== 1) {
5867 				xfr->task_transfer->got_xfr_serial++;
5868 			/* if not first soa, if serial==firstserial, the
5869 			 * third time we are at the end, for IXFR */
5870 			} else if(xfr->task_transfer->incoming_xfr_serial ==
5871 				serial && xfr->task_transfer->got_xfr_serial
5872 				== 2) {
5873 				verbose(VERB_ALGO, "xfr %s: last IXFR packet",
5874 					xfr->task_transfer->master->host);
5875 				*transferdone = 1;
5876 				/* continue parse check, if that succeeds,
5877 				 * transfer is done */
5878 			}
5879 		}
5880 		xfr->task_transfer->rr_scan_num++;
5881 
5882 		/* skip over RR rdata to go to the next RR */
5883 		sldns_buffer_skip(pkt, (ssize_t)rdlen);
5884 	}
5885 
5886 	/* check authority section */
5887 	/* we skip over the RRs checking packet format */
5888 	for(i=0; i<(int)LDNS_NSCOUNT(wire); i++) {
5889 		uint16_t rdlen;
5890 		if(pkt_dname_len(pkt) == 0) {
5891 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5892 				"malformed dname in authority section",
5893 				xfr->task_transfer->master->host);
5894 			return 0;
5895 		}
5896 		if(sldns_buffer_remaining(pkt) < 10) {
5897 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5898 				"truncated RR",
5899 				xfr->task_transfer->master->host);
5900 			return 0;
5901 		}
5902 		(void)sldns_buffer_read_u16(pkt); /* type */
5903 		(void)sldns_buffer_read_u16(pkt); /* class */
5904 		(void)sldns_buffer_read_u32(pkt); /* ttl */
5905 		rdlen = sldns_buffer_read_u16(pkt);
5906 		if(sldns_buffer_remaining(pkt) < rdlen) {
5907 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5908 				"truncated RR rdata",
5909 				xfr->task_transfer->master->host);
5910 			return 0;
5911 		}
5912 		/* skip over RR rdata to go to the next RR */
5913 		sldns_buffer_skip(pkt, (ssize_t)rdlen);
5914 	}
5915 
5916 	/* check additional section */
5917 	for(i=0; i<(int)LDNS_ARCOUNT(wire); i++) {
5918 		uint16_t rdlen;
5919 		if(pkt_dname_len(pkt) == 0) {
5920 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5921 				"malformed dname in additional section",
5922 				xfr->task_transfer->master->host);
5923 			return 0;
5924 		}
5925 		if(sldns_buffer_remaining(pkt) < 10) {
5926 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5927 				"truncated RR",
5928 				xfr->task_transfer->master->host);
5929 			return 0;
5930 		}
5931 		(void)sldns_buffer_read_u16(pkt); /* type */
5932 		(void)sldns_buffer_read_u16(pkt); /* class */
5933 		(void)sldns_buffer_read_u32(pkt); /* ttl */
5934 		rdlen = sldns_buffer_read_u16(pkt);
5935 		if(sldns_buffer_remaining(pkt) < rdlen) {
5936 			verbose(VERB_ALGO, "xfr to %s failed, packet with "
5937 				"truncated RR rdata",
5938 				xfr->task_transfer->master->host);
5939 			return 0;
5940 		}
5941 		/* skip over RR rdata to go to the next RR */
5942 		sldns_buffer_skip(pkt, (ssize_t)rdlen);
5943 	}
5944 
5945 	return 1;
5946 }
5947 
5948 /** Link the data from this packet into the worklist of transferred data */
5949 static int
5950 xfer_link_data(sldns_buffer* pkt, struct auth_xfer* xfr)
5951 {
5952 	/* alloc it */
5953 	struct auth_chunk* e;
5954 	e = (struct auth_chunk*)calloc(1, sizeof(*e));
5955 	if(!e) return 0;
5956 	e->next = NULL;
5957 	e->len = sldns_buffer_limit(pkt);
5958 	e->data = memdup(sldns_buffer_begin(pkt), e->len);
5959 	if(!e->data) {
5960 		free(e);
5961 		return 0;
5962 	}
5963 
5964 	/* alloc succeeded, link into list */
5965 	if(!xfr->task_transfer->chunks_first)
5966 		xfr->task_transfer->chunks_first = e;
5967 	if(xfr->task_transfer->chunks_last)
5968 		xfr->task_transfer->chunks_last->next = e;
5969 	xfr->task_transfer->chunks_last = e;
5970 	return 1;
5971 }
5972 
5973 /** task transfer.  the list of data is complete. process it and if failed
5974  * move to next master, if succeeded, end the task transfer */
5975 static void
5976 process_list_end_transfer(struct auth_xfer* xfr, struct module_env* env)
5977 {
5978 	int ixfr_fail = 0;
5979 	if(xfr_process_chunk_list(xfr, env, &ixfr_fail)) {
5980 		/* it worked! */
5981 		auth_chunks_delete(xfr->task_transfer);
5982 
5983 		/* we fetched the zone, move to wait task */
5984 		xfr_transfer_disown(xfr);
5985 
5986 		if(xfr->notify_received && (!xfr->notify_has_serial ||
5987 			(xfr->notify_has_serial &&
5988 			xfr_serial_means_update(xfr, xfr->notify_serial)))) {
5989 			uint32_t sr = xfr->notify_serial;
5990 			int has_sr = xfr->notify_has_serial;
5991 			/* we received a notify while probe/transfer was
5992 			 * in progress.  start a new probe and transfer */
5993 			xfr->notify_received = 0;
5994 			xfr->notify_has_serial = 0;
5995 			xfr->notify_serial = 0;
5996 			if(!xfr_start_probe(xfr, env, NULL)) {
5997 				/* if we couldn't start it, already in
5998 				 * progress; restore notify serial,
5999 				 * while xfr still locked */
6000 				xfr->notify_received = 1;
6001 				xfr->notify_has_serial = has_sr;
6002 				xfr->notify_serial = sr;
6003 				lock_basic_unlock(&xfr->lock);
6004 			}
6005 			return;
6006 		} else {
6007 			/* pick up the nextprobe task and wait (normail wait time) */
6008 			if(xfr->task_nextprobe->worker == NULL)
6009 				xfr_set_timeout(xfr, env, 0, 0);
6010 		}
6011 		lock_basic_unlock(&xfr->lock);
6012 		return;
6013 	}
6014 	/* processing failed */
6015 	/* when done, delete data from list */
6016 	auth_chunks_delete(xfr->task_transfer);
6017 	if(ixfr_fail) {
6018 		xfr->task_transfer->ixfr_fail = 1;
6019 	} else {
6020 		xfr_transfer_nextmaster(xfr);
6021 	}
6022 	xfr_transfer_nexttarget_or_end(xfr, env);
6023 }
6024 
6025 /** callback for the task_transfer timer */
6026 void
6027 auth_xfer_transfer_timer_callback(void* arg)
6028 {
6029 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
6030 	struct module_env* env;
6031 	int gonextonfail = 1;
6032 	log_assert(xfr->task_transfer);
6033 	lock_basic_lock(&xfr->lock);
6034 	env = xfr->task_transfer->env;
6035 	if(!env || env->outnet->want_to_quit) {
6036 		lock_basic_unlock(&xfr->lock);
6037 		return; /* stop on quit */
6038 	}
6039 
6040 	verbose(VERB_ALGO, "xfr stopped, connection timeout to %s",
6041 		xfr->task_transfer->master->host);
6042 
6043 	/* see if IXFR caused the failure, if so, try AXFR */
6044 	if(xfr->task_transfer->on_ixfr) {
6045 		xfr->task_transfer->ixfr_possible_timeout_count++;
6046 		if(xfr->task_transfer->ixfr_possible_timeout_count >=
6047 			NUM_TIMEOUTS_FALLBACK_IXFR) {
6048 			verbose(VERB_ALGO, "xfr to %s, fallback "
6049 				"from IXFR to AXFR (because of timeouts)",
6050 				xfr->task_transfer->master->host);
6051 			xfr->task_transfer->ixfr_fail = 1;
6052 			gonextonfail = 0;
6053 		}
6054 	}
6055 
6056 	/* delete transferred data from list */
6057 	auth_chunks_delete(xfr->task_transfer);
6058 	comm_point_delete(xfr->task_transfer->cp);
6059 	xfr->task_transfer->cp = NULL;
6060 	if(gonextonfail)
6061 		xfr_transfer_nextmaster(xfr);
6062 	xfr_transfer_nexttarget_or_end(xfr, env);
6063 }
6064 
6065 /** callback for task_transfer tcp connections */
6066 int
6067 auth_xfer_transfer_tcp_callback(struct comm_point* c, void* arg, int err,
6068 	struct comm_reply* ATTR_UNUSED(repinfo))
6069 {
6070 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
6071 	struct module_env* env;
6072 	int gonextonfail = 1;
6073 	int transferdone = 0;
6074 	log_assert(xfr->task_transfer);
6075 	lock_basic_lock(&xfr->lock);
6076 	env = xfr->task_transfer->env;
6077 	if(!env || env->outnet->want_to_quit) {
6078 		lock_basic_unlock(&xfr->lock);
6079 		return 0; /* stop on quit */
6080 	}
6081 	/* stop the timer */
6082 	comm_timer_disable(xfr->task_transfer->timer);
6083 
6084 	if(err != NETEVENT_NOERROR) {
6085 		/* connection failed, closed, or timeout */
6086 		/* stop this transfer, cleanup
6087 		 * and continue task_transfer*/
6088 		verbose(VERB_ALGO, "xfr stopped, connection lost to %s",
6089 			xfr->task_transfer->master->host);
6090 
6091 		/* see if IXFR caused the failure, if so, try AXFR */
6092 		if(xfr->task_transfer->on_ixfr) {
6093 			xfr->task_transfer->ixfr_possible_timeout_count++;
6094 			if(xfr->task_transfer->ixfr_possible_timeout_count >=
6095 				NUM_TIMEOUTS_FALLBACK_IXFR) {
6096 				verbose(VERB_ALGO, "xfr to %s, fallback "
6097 					"from IXFR to AXFR (because of timeouts)",
6098 					xfr->task_transfer->master->host);
6099 				xfr->task_transfer->ixfr_fail = 1;
6100 				gonextonfail = 0;
6101 			}
6102 		}
6103 
6104 	failed:
6105 		/* delete transferred data from list */
6106 		auth_chunks_delete(xfr->task_transfer);
6107 		comm_point_delete(xfr->task_transfer->cp);
6108 		xfr->task_transfer->cp = NULL;
6109 		if(gonextonfail)
6110 			xfr_transfer_nextmaster(xfr);
6111 		xfr_transfer_nexttarget_or_end(xfr, env);
6112 		return 0;
6113 	}
6114 	/* note that IXFR worked without timeout */
6115 	if(xfr->task_transfer->on_ixfr)
6116 		xfr->task_transfer->ixfr_possible_timeout_count = 0;
6117 
6118 	/* handle returned packet */
6119 	/* if it fails, cleanup and end this transfer */
6120 	/* if it needs to fallback from IXFR to AXFR, do that */
6121 	if(!check_xfer_packet(c->buffer, xfr, &gonextonfail, &transferdone)) {
6122 		goto failed;
6123 	}
6124 	/* if it is good, link it into the list of data */
6125 	/* if the link into list of data fails (malloc fail) cleanup and end */
6126 	if(!xfer_link_data(c->buffer, xfr)) {
6127 		verbose(VERB_ALGO, "xfr stopped to %s, malloc failed",
6128 			xfr->task_transfer->master->host);
6129 		goto failed;
6130 	}
6131 	/* if the transfer is done now, disconnect and process the list */
6132 	if(transferdone) {
6133 		comm_point_delete(xfr->task_transfer->cp);
6134 		xfr->task_transfer->cp = NULL;
6135 		process_list_end_transfer(xfr, env);
6136 		return 0;
6137 	}
6138 
6139 	/* if we want to read more messages, setup the commpoint to read
6140 	 * a DNS packet, and the timeout */
6141 	lock_basic_unlock(&xfr->lock);
6142 	c->tcp_is_reading = 1;
6143 	sldns_buffer_clear(c->buffer);
6144 	comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
6145 	return 0;
6146 }
6147 
6148 /** callback for task_transfer http connections */
6149 int
6150 auth_xfer_transfer_http_callback(struct comm_point* c, void* arg, int err,
6151 	struct comm_reply* repinfo)
6152 {
6153 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
6154 	struct module_env* env;
6155 	log_assert(xfr->task_transfer);
6156 	lock_basic_lock(&xfr->lock);
6157 	env = xfr->task_transfer->env;
6158 	if(!env || env->outnet->want_to_quit) {
6159 		lock_basic_unlock(&xfr->lock);
6160 		return 0; /* stop on quit */
6161 	}
6162 	verbose(VERB_ALGO, "auth zone transfer http callback");
6163 	/* stop the timer */
6164 	comm_timer_disable(xfr->task_transfer->timer);
6165 
6166 	if(err != NETEVENT_NOERROR && err != NETEVENT_DONE) {
6167 		/* connection failed, closed, or timeout */
6168 		/* stop this transfer, cleanup
6169 		 * and continue task_transfer*/
6170 		verbose(VERB_ALGO, "http stopped, connection lost to %s",
6171 			xfr->task_transfer->master->host);
6172 	failed:
6173 		/* delete transferred data from list */
6174 		auth_chunks_delete(xfr->task_transfer);
6175 		if(repinfo) repinfo->c = NULL; /* signal cp deleted to
6176 				the routine calling this callback */
6177 		comm_point_delete(xfr->task_transfer->cp);
6178 		xfr->task_transfer->cp = NULL;
6179 		xfr_transfer_nextmaster(xfr);
6180 		xfr_transfer_nexttarget_or_end(xfr, env);
6181 		return 0;
6182 	}
6183 
6184 	/* if it is good, link it into the list of data */
6185 	/* if the link into list of data fails (malloc fail) cleanup and end */
6186 	if(sldns_buffer_limit(c->buffer) > 0) {
6187 		verbose(VERB_ALGO, "auth zone http queued up %d bytes",
6188 			(int)sldns_buffer_limit(c->buffer));
6189 		if(!xfer_link_data(c->buffer, xfr)) {
6190 			verbose(VERB_ALGO, "http stopped to %s, malloc failed",
6191 				xfr->task_transfer->master->host);
6192 			goto failed;
6193 		}
6194 	}
6195 	/* if the transfer is done now, disconnect and process the list */
6196 	if(err == NETEVENT_DONE) {
6197 		if(repinfo) repinfo->c = NULL; /* signal cp deleted to
6198 				the routine calling this callback */
6199 		comm_point_delete(xfr->task_transfer->cp);
6200 		xfr->task_transfer->cp = NULL;
6201 		process_list_end_transfer(xfr, env);
6202 		return 0;
6203 	}
6204 
6205 	/* if we want to read more messages, setup the commpoint to read
6206 	 * a DNS packet, and the timeout */
6207 	lock_basic_unlock(&xfr->lock);
6208 	c->tcp_is_reading = 1;
6209 	sldns_buffer_clear(c->buffer);
6210 	comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
6211 	return 0;
6212 }
6213 
6214 
6215 /** start transfer task by this worker , xfr is locked. */
6216 static void
6217 xfr_start_transfer(struct auth_xfer* xfr, struct module_env* env,
6218 	struct auth_master* master)
6219 {
6220 	log_assert(xfr->task_transfer != NULL);
6221 	log_assert(xfr->task_transfer->worker == NULL);
6222 	log_assert(xfr->task_transfer->chunks_first == NULL);
6223 	log_assert(xfr->task_transfer->chunks_last == NULL);
6224 	xfr->task_transfer->worker = env->worker;
6225 	xfr->task_transfer->env = env;
6226 
6227 	/* init transfer process */
6228 	/* find that master in the transfer's list of masters? */
6229 	xfr_transfer_start_list(xfr, master);
6230 	/* start lookup for hostnames in transfer master list */
6231 	xfr_transfer_start_lookups(xfr);
6232 
6233 	/* initiate TCP, and set timeout on it */
6234 	xfr_transfer_nexttarget_or_end(xfr, env);
6235 }
6236 
6237 /** disown task_probe.  caller must hold xfr.lock */
6238 static void
6239 xfr_probe_disown(struct auth_xfer* xfr)
6240 {
6241 	/* remove timer (from this worker's event base) */
6242 	comm_timer_delete(xfr->task_probe->timer);
6243 	xfr->task_probe->timer = NULL;
6244 	/* remove the commpoint */
6245 	comm_point_delete(xfr->task_probe->cp);
6246 	xfr->task_probe->cp = NULL;
6247 	/* we don't own this item anymore */
6248 	xfr->task_probe->worker = NULL;
6249 	xfr->task_probe->env = NULL;
6250 }
6251 
6252 /** send the UDP probe to the master, this is part of task_probe */
6253 static int
6254 xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
6255 	int timeout)
6256 {
6257 	struct sockaddr_storage addr;
6258 	socklen_t addrlen = 0;
6259 	struct timeval t;
6260 	/* pick master */
6261 	struct auth_master* master = xfr_probe_current_master(xfr);
6262 	char *auth_name = NULL;
6263 	if(!master) return 0;
6264 	if(master->allow_notify) return 0; /* only for notify */
6265 	if(master->http) return 0; /* only masters get SOA UDP probe,
6266 		not urls, if those are in this list */
6267 
6268 	/* get master addr */
6269 	if(xfr->task_probe->scan_addr) {
6270 		addrlen = xfr->task_probe->scan_addr->addrlen;
6271 		memmove(&addr, &xfr->task_probe->scan_addr->addr, addrlen);
6272 	} else {
6273 		if(!authextstrtoaddr(master->host, &addr, &addrlen, &auth_name)) {
6274 			/* the ones that are not in addr format are supposed
6275 			 * to be looked up.  The lookup has failed however,
6276 			 * so skip them */
6277 			char zname[255+1];
6278 			dname_str(xfr->name, zname);
6279 			log_err("%s: failed lookup, cannot probe to master %s",
6280 				zname, master->host);
6281 			return 0;
6282 		}
6283 		if (auth_name != NULL) {
6284 			if (addr.ss_family == AF_INET
6285 			&&  (int)ntohs(((struct sockaddr_in *)&addr)->sin_port)
6286 		            == env->cfg->ssl_port)
6287 				((struct sockaddr_in *)&addr)->sin_port
6288 					= htons((uint16_t)env->cfg->port);
6289 			else if (addr.ss_family == AF_INET6
6290 			&&  (int)ntohs(((struct sockaddr_in6 *)&addr)->sin6_port)
6291 		            == env->cfg->ssl_port)
6292                         	((struct sockaddr_in6 *)&addr)->sin6_port
6293 					= htons((uint16_t)env->cfg->port);
6294 		}
6295 	}
6296 
6297 	/* create packet */
6298 	/* create new ID for new probes, but not on timeout retries,
6299 	 * this means we'll accept replies to previous retries to same ip */
6300 	if(timeout == AUTH_PROBE_TIMEOUT)
6301 		xfr->task_probe->id = GET_RANDOM_ID(env->rnd);
6302 	xfr_create_soa_probe_packet(xfr, env->scratch_buffer,
6303 		xfr->task_probe->id);
6304 	/* we need to remove the cp if we have a different ip4/ip6 type now */
6305 	if(xfr->task_probe->cp &&
6306 		((xfr->task_probe->cp_is_ip6 && !addr_is_ip6(&addr, addrlen)) ||
6307 		(!xfr->task_probe->cp_is_ip6 && addr_is_ip6(&addr, addrlen)))
6308 		) {
6309 		comm_point_delete(xfr->task_probe->cp);
6310 		xfr->task_probe->cp = NULL;
6311 	}
6312 	if(!xfr->task_probe->cp) {
6313 		if(addr_is_ip6(&addr, addrlen))
6314 			xfr->task_probe->cp_is_ip6 = 1;
6315 		else 	xfr->task_probe->cp_is_ip6 = 0;
6316 		xfr->task_probe->cp = outnet_comm_point_for_udp(env->outnet,
6317 			auth_xfer_probe_udp_callback, xfr, &addr, addrlen);
6318 		if(!xfr->task_probe->cp) {
6319 			char zname[255+1], as[256];
6320 			dname_str(xfr->name, zname);
6321 			addr_to_str(&addr, addrlen, as, sizeof(as));
6322 			verbose(VERB_ALGO, "cannot create udp cp for "
6323 				"probe %s to %s", zname, as);
6324 			return 0;
6325 		}
6326 	}
6327 	if(!xfr->task_probe->timer) {
6328 		xfr->task_probe->timer = comm_timer_create(env->worker_base,
6329 			auth_xfer_probe_timer_callback, xfr);
6330 		if(!xfr->task_probe->timer) {
6331 			log_err("malloc failure");
6332 			return 0;
6333 		}
6334 	}
6335 
6336 	/* send udp packet */
6337 	if(!comm_point_send_udp_msg(xfr->task_probe->cp, env->scratch_buffer,
6338 		(struct sockaddr*)&addr, addrlen, 0)) {
6339 		char zname[255+1], as[256];
6340 		dname_str(xfr->name, zname);
6341 		addr_to_str(&addr, addrlen, as, sizeof(as));
6342 		verbose(VERB_ALGO, "failed to send soa probe for %s to %s",
6343 			zname, as);
6344 		return 0;
6345 	}
6346 	if(verbosity >= VERB_ALGO) {
6347 		char zname[255+1], as[256];
6348 		dname_str(xfr->name, zname);
6349 		addr_to_str(&addr, addrlen, as, sizeof(as));
6350 		verbose(VERB_ALGO, "auth zone %s soa probe sent to %s", zname,
6351 			as);
6352 	}
6353 	xfr->task_probe->timeout = timeout;
6354 #ifndef S_SPLINT_S
6355 	t.tv_sec = timeout/1000;
6356 	t.tv_usec = (timeout%1000)*1000;
6357 #endif
6358 	comm_timer_set(xfr->task_probe->timer, &t);
6359 
6360 	return 1;
6361 }
6362 
6363 /** callback for task_probe timer */
6364 void
6365 auth_xfer_probe_timer_callback(void* arg)
6366 {
6367 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
6368 	struct module_env* env;
6369 	log_assert(xfr->task_probe);
6370 	lock_basic_lock(&xfr->lock);
6371 	env = xfr->task_probe->env;
6372 	if(!env || env->outnet->want_to_quit) {
6373 		lock_basic_unlock(&xfr->lock);
6374 		return; /* stop on quit */
6375 	}
6376 
6377 	if(verbosity >= VERB_ALGO) {
6378 		char zname[255+1];
6379 		dname_str(xfr->name, zname);
6380 		verbose(VERB_ALGO, "auth zone %s soa probe timeout", zname);
6381 	}
6382 	if(xfr->task_probe->timeout <= AUTH_PROBE_TIMEOUT_STOP) {
6383 		/* try again with bigger timeout */
6384 		if(xfr_probe_send_probe(xfr, env, xfr->task_probe->timeout*2)) {
6385 			lock_basic_unlock(&xfr->lock);
6386 			return;
6387 		}
6388 	}
6389 	/* delete commpoint so a new one is created, with a fresh port nr */
6390 	comm_point_delete(xfr->task_probe->cp);
6391 	xfr->task_probe->cp = NULL;
6392 
6393 	/* too many timeouts (or fail to send), move to next or end */
6394 	xfr_probe_nextmaster(xfr);
6395 	xfr_probe_send_or_end(xfr, env);
6396 }
6397 
6398 /** callback for task_probe udp packets */
6399 int
6400 auth_xfer_probe_udp_callback(struct comm_point* c, void* arg, int err,
6401 	struct comm_reply* repinfo)
6402 {
6403 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
6404 	struct module_env* env;
6405 	log_assert(xfr->task_probe);
6406 	lock_basic_lock(&xfr->lock);
6407 	env = xfr->task_probe->env;
6408 	if(!env || env->outnet->want_to_quit) {
6409 		lock_basic_unlock(&xfr->lock);
6410 		return 0; /* stop on quit */
6411 	}
6412 
6413 	/* the comm_point_udp_callback is in a for loop for NUM_UDP_PER_SELECT
6414 	 * and we set rep.c=NULL to stop if from looking inside the commpoint*/
6415 	repinfo->c = NULL;
6416 	/* stop the timer */
6417 	comm_timer_disable(xfr->task_probe->timer);
6418 
6419 	/* see if we got a packet and what that means */
6420 	if(err == NETEVENT_NOERROR) {
6421 		uint32_t serial = 0;
6422 		if(check_packet_ok(c->buffer, LDNS_RR_TYPE_SOA, xfr,
6423 			&serial)) {
6424 			/* successful lookup */
6425 			if(verbosity >= VERB_ALGO) {
6426 				char buf[256];
6427 				dname_str(xfr->name, buf);
6428 				verbose(VERB_ALGO, "auth zone %s: soa probe "
6429 					"serial is %u", buf, (unsigned)serial);
6430 			}
6431 			/* see if this serial indicates that the zone has
6432 			 * to be updated */
6433 			if(xfr_serial_means_update(xfr, serial)) {
6434 				/* if updated, start the transfer task, if needed */
6435 				verbose(VERB_ALGO, "auth_zone updated, start transfer");
6436 				if(xfr->task_transfer->worker == NULL) {
6437 					struct auth_master* master =
6438 						xfr_probe_current_master(xfr);
6439 					/* if we have download URLs use them
6440 					 * in preference to this master we
6441 					 * just probed the SOA from */
6442 					if(xfr->task_transfer->masters &&
6443 						xfr->task_transfer->masters->http)
6444 						master = NULL;
6445 					xfr_probe_disown(xfr);
6446 					xfr_start_transfer(xfr, env, master);
6447 					return 0;
6448 
6449 				}
6450 				/* other tasks are running, we don't do this anymore */
6451 				xfr_probe_disown(xfr);
6452 				lock_basic_unlock(&xfr->lock);
6453 				/* return, we don't sent a reply to this udp packet,
6454 				 * and we setup the tasks to do next */
6455 				return 0;
6456 			} else {
6457 				verbose(VERB_ALGO, "auth_zone master reports unchanged soa serial");
6458 				/* we if cannot find updates amongst the
6459 				 * masters, this means we then have a new lease
6460 				 * on the zone */
6461 				xfr->task_probe->have_new_lease = 1;
6462 			}
6463 		} else {
6464 			if(verbosity >= VERB_ALGO) {
6465 				char buf[256];
6466 				dname_str(xfr->name, buf);
6467 				verbose(VERB_ALGO, "auth zone %s: bad reply to soa probe", buf);
6468 			}
6469 		}
6470 	} else {
6471 		if(verbosity >= VERB_ALGO) {
6472 			char buf[256];
6473 			dname_str(xfr->name, buf);
6474 			verbose(VERB_ALGO, "auth zone %s: soa probe failed", buf);
6475 		}
6476 	}
6477 
6478 	/* failed lookup or not an update */
6479 	/* delete commpoint so a new one is created, with a fresh port nr */
6480 	comm_point_delete(xfr->task_probe->cp);
6481 	xfr->task_probe->cp = NULL;
6482 
6483 	/* if the result was not a successfull probe, we need
6484 	 * to send the next one */
6485 	xfr_probe_nextmaster(xfr);
6486 	xfr_probe_send_or_end(xfr, env);
6487 	return 0;
6488 }
6489 
6490 /** lookup a host name for its addresses, if needed */
6491 static int
6492 xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
6493 {
6494 	struct sockaddr_storage addr;
6495 	socklen_t addrlen = 0;
6496 	struct auth_master* master = xfr->task_probe->lookup_target;
6497 	struct query_info qinfo;
6498 	uint16_t qflags = BIT_RD;
6499 	uint8_t dname[LDNS_MAX_DOMAINLEN+1];
6500 	struct edns_data edns;
6501 	sldns_buffer* buf = env->scratch_buffer;
6502 	if(!master) return 0;
6503 	if(extstrtoaddr(master->host, &addr, &addrlen)) {
6504 		/* not needed, host is in IP addr format */
6505 		return 0;
6506 	}
6507 	if(master->allow_notify && !master->http &&
6508 		strchr(master->host, '/') != NULL &&
6509 		strchr(master->host, '/') == strrchr(master->host, '/')) {
6510 		return 0; /* is IP/prefix format, not something to look up */
6511 	}
6512 
6513 	/* use mesh_new_callback to probe for non-addr hosts,
6514 	 * and then wait for them to be looked up (in cache, or query) */
6515 	qinfo.qname_len = sizeof(dname);
6516 	if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
6517 		!= 0) {
6518 		log_err("cannot parse host name of master %s", master->host);
6519 		return 0;
6520 	}
6521 	qinfo.qname = dname;
6522 	qinfo.qclass = xfr->dclass;
6523 	qinfo.qtype = LDNS_RR_TYPE_A;
6524 	if(xfr->task_probe->lookup_aaaa)
6525 		qinfo.qtype = LDNS_RR_TYPE_AAAA;
6526 	qinfo.local_alias = NULL;
6527 	if(verbosity >= VERB_ALGO) {
6528 		char buf1[512];
6529 		char buf2[LDNS_MAX_DOMAINLEN+1];
6530 		dname_str(xfr->name, buf2);
6531 		snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
6532 			" for task_probe", buf2);
6533 		log_query_info(VERB_ALGO, buf1, &qinfo);
6534 	}
6535 	edns.edns_present = 1;
6536 	edns.ext_rcode = 0;
6537 	edns.edns_version = 0;
6538 	edns.bits = EDNS_DO;
6539 	edns.opt_list = NULL;
6540 	edns.padding_block_size = 0;
6541 	if(sldns_buffer_capacity(buf) < 65535)
6542 		edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
6543 	else	edns.udp_size = 65535;
6544 
6545 	/* unlock xfr during mesh_new_callback() because the callback can be
6546 	 * called straight away */
6547 	lock_basic_unlock(&xfr->lock);
6548 	if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
6549 		&auth_xfer_probe_lookup_callback, xfr)) {
6550 		lock_basic_lock(&xfr->lock);
6551 		log_err("out of memory lookup up master %s", master->host);
6552 		return 0;
6553 	}
6554 	lock_basic_lock(&xfr->lock);
6555 	return 1;
6556 }
6557 
6558 /** move to sending the probe packets, next if fails. task_probe */
6559 static void
6560 xfr_probe_send_or_end(struct auth_xfer* xfr, struct module_env* env)
6561 {
6562 	/* are we doing hostname lookups? */
6563 	while(xfr->task_probe->lookup_target) {
6564 		if(xfr_probe_lookup_host(xfr, env)) {
6565 			/* wait for lookup to finish,
6566 			 * note that the hostname may be in unbound's cache
6567 			 * and we may then get an instant cache response,
6568 			 * and that calls the callback just like a full
6569 			 * lookup and lookup failures also call callback */
6570 			if(verbosity >= VERB_ALGO) {
6571 				char zname[255+1];
6572 				dname_str(xfr->name, zname);
6573 				verbose(VERB_ALGO, "auth zone %s probe next target lookup", zname);
6574 			}
6575 			lock_basic_unlock(&xfr->lock);
6576 			return;
6577 		}
6578 		xfr_probe_move_to_next_lookup(xfr, env);
6579 	}
6580 	/* probe of list has ended.  Create or refresh the list of of
6581 	 * allow_notify addrs */
6582 	probe_copy_masters_for_allow_notify(xfr);
6583 	if(verbosity >= VERB_ALGO) {
6584 		char zname[255+1];
6585 		dname_str(xfr->name, zname);
6586 		verbose(VERB_ALGO, "auth zone %s probe: notify addrs updated", zname);
6587 	}
6588 	if(xfr->task_probe->only_lookup) {
6589 		/* only wanted lookups for copy, stop probe and start wait */
6590 		xfr->task_probe->only_lookup = 0;
6591 		if(verbosity >= VERB_ALGO) {
6592 			char zname[255+1];
6593 			dname_str(xfr->name, zname);
6594 			verbose(VERB_ALGO, "auth zone %s probe: finished only_lookup", zname);
6595 		}
6596 		xfr_probe_disown(xfr);
6597 		if(xfr->task_nextprobe->worker == NULL)
6598 			xfr_set_timeout(xfr, env, 0, 0);
6599 		lock_basic_unlock(&xfr->lock);
6600 		return;
6601 	}
6602 
6603 	/* send probe packets */
6604 	while(!xfr_probe_end_of_list(xfr)) {
6605 		if(xfr_probe_send_probe(xfr, env, AUTH_PROBE_TIMEOUT)) {
6606 			/* successfully sent probe, wait for callback */
6607 			lock_basic_unlock(&xfr->lock);
6608 			return;
6609 		}
6610 		/* failed to send probe, next master */
6611 		xfr_probe_nextmaster(xfr);
6612 	}
6613 
6614 	/* done with probe sequence, wait */
6615 	if(xfr->task_probe->have_new_lease) {
6616 		/* if zone not updated, start the wait timer again */
6617 		if(verbosity >= VERB_ALGO) {
6618 			char zname[255+1];
6619 			dname_str(xfr->name, zname);
6620 			verbose(VERB_ALGO, "auth_zone %s unchanged, new lease, wait", zname);
6621 		}
6622 		xfr_probe_disown(xfr);
6623 		if(xfr->have_zone)
6624 			xfr->lease_time = *env->now;
6625 		if(xfr->task_nextprobe->worker == NULL)
6626 			xfr_set_timeout(xfr, env, 0, 0);
6627 	} else {
6628 		if(verbosity >= VERB_ALGO) {
6629 			char zname[255+1];
6630 			dname_str(xfr->name, zname);
6631 			verbose(VERB_ALGO, "auth zone %s soa probe failed, wait to retry", zname);
6632 		}
6633 		/* we failed to send this as well, move to the wait task,
6634 		 * use the shorter retry timeout */
6635 		xfr_probe_disown(xfr);
6636 		/* pick up the nextprobe task and wait */
6637 		if(xfr->task_nextprobe->worker == NULL)
6638 			xfr_set_timeout(xfr, env, 1, 0);
6639 	}
6640 
6641 	lock_basic_unlock(&xfr->lock);
6642 }
6643 
6644 /** callback for task_probe lookup of host name, of A or AAAA */
6645 void auth_xfer_probe_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
6646 	enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
6647 	int ATTR_UNUSED(was_ratelimited))
6648 {
6649 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
6650 	struct module_env* env;
6651 	log_assert(xfr->task_probe);
6652 	lock_basic_lock(&xfr->lock);
6653 	env = xfr->task_probe->env;
6654 	if(!env || env->outnet->want_to_quit) {
6655 		lock_basic_unlock(&xfr->lock);
6656 		return; /* stop on quit */
6657 	}
6658 
6659 	/* process result */
6660 	if(rcode == LDNS_RCODE_NOERROR) {
6661 		uint16_t wanted_qtype = LDNS_RR_TYPE_A;
6662 		struct regional* temp = env->scratch;
6663 		struct query_info rq;
6664 		struct reply_info* rep;
6665 		if(xfr->task_probe->lookup_aaaa)
6666 			wanted_qtype = LDNS_RR_TYPE_AAAA;
6667 		memset(&rq, 0, sizeof(rq));
6668 		rep = parse_reply_in_temp_region(buf, temp, &rq);
6669 		if(rep && rq.qtype == wanted_qtype &&
6670 			FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
6671 			/* parsed successfully */
6672 			struct ub_packed_rrset_key* answer =
6673 				reply_find_answer_rrset(&rq, rep);
6674 			if(answer) {
6675 				xfr_master_add_addrs(xfr->task_probe->
6676 					lookup_target, answer, wanted_qtype);
6677 			} else {
6678 				if(verbosity >= VERB_ALGO) {
6679 					char zname[255+1];
6680 					dname_str(xfr->name, zname);
6681 					verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup has nodata", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6682 				}
6683 			}
6684 		} else {
6685 			if(verbosity >= VERB_ALGO) {
6686 				char zname[255+1];
6687 				dname_str(xfr->name, zname);
6688 				verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup has no address", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6689 			}
6690 		}
6691 		regional_free_all(temp);
6692 	} else {
6693 		if(verbosity >= VERB_ALGO) {
6694 			char zname[255+1];
6695 			dname_str(xfr->name, zname);
6696 			verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup failed", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6697 		}
6698 	}
6699 	if(xfr->task_probe->lookup_target->list &&
6700 		xfr->task_probe->lookup_target == xfr_probe_current_master(xfr))
6701 		xfr->task_probe->scan_addr = xfr->task_probe->lookup_target->list;
6702 
6703 	/* move to lookup AAAA after A lookup, move to next hostname lookup,
6704 	 * or move to send the probes, or, if nothing to do, end task_probe */
6705 	xfr_probe_move_to_next_lookup(xfr, env);
6706 	xfr_probe_send_or_end(xfr, env);
6707 }
6708 
6709 /** disown task_nextprobe.  caller must hold xfr.lock */
6710 static void
6711 xfr_nextprobe_disown(struct auth_xfer* xfr)
6712 {
6713 	/* delete the timer, because the next worker to pick this up may
6714 	 * not have the same event base */
6715 	comm_timer_delete(xfr->task_nextprobe->timer);
6716 	xfr->task_nextprobe->timer = NULL;
6717 	xfr->task_nextprobe->next_probe = 0;
6718 	/* we don't own this item anymore */
6719 	xfr->task_nextprobe->worker = NULL;
6720 	xfr->task_nextprobe->env = NULL;
6721 }
6722 
6723 /** xfer nextprobe timeout callback, this is part of task_nextprobe */
6724 void
6725 auth_xfer_timer(void* arg)
6726 {
6727 	struct auth_xfer* xfr = (struct auth_xfer*)arg;
6728 	struct module_env* env;
6729 	log_assert(xfr->task_nextprobe);
6730 	lock_basic_lock(&xfr->lock);
6731 	env = xfr->task_nextprobe->env;
6732 	if(!env || env->outnet->want_to_quit) {
6733 		lock_basic_unlock(&xfr->lock);
6734 		return; /* stop on quit */
6735 	}
6736 
6737 	/* see if zone has expired, and if so, also set auth_zone expired */
6738 	if(xfr->have_zone && !xfr->zone_expired &&
6739 	   *env->now >= xfr->lease_time + xfr->expiry) {
6740 		lock_basic_unlock(&xfr->lock);
6741 		auth_xfer_set_expired(xfr, env, 1);
6742 		lock_basic_lock(&xfr->lock);
6743 	}
6744 
6745 	xfr_nextprobe_disown(xfr);
6746 
6747 	if(!xfr_start_probe(xfr, env, NULL)) {
6748 		/* not started because already in progress */
6749 		lock_basic_unlock(&xfr->lock);
6750 	}
6751 }
6752 
6753 /** return true if there are probe (SOA UDP query) targets in the master list*/
6754 static int
6755 have_probe_targets(struct auth_master* list)
6756 {
6757 	struct auth_master* p;
6758 	for(p=list; p; p = p->next) {
6759 		if(!p->allow_notify && p->host)
6760 			return 1;
6761 	}
6762 	return 0;
6763 }
6764 
6765 /** start task_probe if possible, if no masters for probe start task_transfer
6766  * returns true if task has been started, and false if the task is already
6767  * in progress. */
6768 static int
6769 xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
6770 	struct auth_master* spec)
6771 {
6772 	/* see if we need to start a probe (or maybe it is already in
6773 	 * progress (due to notify)) */
6774 	if(xfr->task_probe->worker == NULL) {
6775 		if(!have_probe_targets(xfr->task_probe->masters) &&
6776 			!(xfr->task_probe->only_lookup &&
6777 			xfr->task_probe->masters != NULL)) {
6778 			/* useless to pick up task_probe, no masters to
6779 			 * probe. Instead attempt to pick up task transfer */
6780 			if(xfr->task_transfer->worker == NULL) {
6781 				xfr_start_transfer(xfr, env, spec);
6782 				return 1;
6783 			}
6784 			/* task transfer already in progress */
6785 			return 0;
6786 		}
6787 
6788 		/* pick up the probe task ourselves */
6789 		xfr->task_probe->worker = env->worker;
6790 		xfr->task_probe->env = env;
6791 		xfr->task_probe->cp = NULL;
6792 
6793 		/* start the task */
6794 		/* have not seen a new lease yet, this scan */
6795 		xfr->task_probe->have_new_lease = 0;
6796 		/* if this was a timeout, no specific first master to scan */
6797 		/* otherwise, spec is nonNULL the notified master, scan
6798 		 * first and also transfer first from it */
6799 		xfr_probe_start_list(xfr, spec);
6800 		/* setup to start the lookup of hostnames of masters afresh */
6801 		xfr_probe_start_lookups(xfr);
6802 		/* send the probe packet or next send, or end task */
6803 		xfr_probe_send_or_end(xfr, env);
6804 		return 1;
6805 	}
6806 	return 0;
6807 }
6808 
6809 /** for task_nextprobe.
6810  * determine next timeout for auth_xfer. Also (re)sets timer.
6811  * @param xfr: task structure
6812  * @param env: module environment, with worker and time.
6813  * @param failure: set true if timer should be set for failure retry.
6814  * @param lookup_only: only perform lookups when timer done, 0 sec timeout
6815  */
6816 static void
6817 xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
6818 	int failure, int lookup_only)
6819 {
6820 	struct timeval tv;
6821 	log_assert(xfr->task_nextprobe != NULL);
6822 	log_assert(xfr->task_nextprobe->worker == NULL ||
6823 		xfr->task_nextprobe->worker == env->worker);
6824 	/* normally, nextprobe = startoflease + refresh,
6825 	 * but if expiry is sooner, use that one.
6826 	 * after a failure, use the retry timer instead. */
6827 	xfr->task_nextprobe->next_probe = *env->now;
6828 	if(xfr->lease_time && !failure)
6829 		xfr->task_nextprobe->next_probe = xfr->lease_time;
6830 
6831 	if(!failure) {
6832 		xfr->task_nextprobe->backoff = 0;
6833 	} else {
6834 		if(xfr->task_nextprobe->backoff == 0)
6835 				xfr->task_nextprobe->backoff = 3;
6836 		else	xfr->task_nextprobe->backoff *= 2;
6837 		if(xfr->task_nextprobe->backoff > AUTH_TRANSFER_MAX_BACKOFF)
6838 			xfr->task_nextprobe->backoff =
6839 				AUTH_TRANSFER_MAX_BACKOFF;
6840 	}
6841 
6842 	if(xfr->have_zone) {
6843 		time_t wait = xfr->refresh;
6844 		if(failure) wait = xfr->retry;
6845 		if(xfr->expiry < wait)
6846 			xfr->task_nextprobe->next_probe += xfr->expiry;
6847 		else	xfr->task_nextprobe->next_probe += wait;
6848 		if(failure)
6849 			xfr->task_nextprobe->next_probe +=
6850 				xfr->task_nextprobe->backoff;
6851 		/* put the timer exactly on expiry, if possible */
6852 		if(xfr->lease_time && xfr->lease_time+xfr->expiry <
6853 			xfr->task_nextprobe->next_probe &&
6854 			xfr->lease_time+xfr->expiry > *env->now)
6855 			xfr->task_nextprobe->next_probe =
6856 				xfr->lease_time+xfr->expiry;
6857 	} else {
6858 		xfr->task_nextprobe->next_probe +=
6859 			xfr->task_nextprobe->backoff;
6860 	}
6861 
6862 	if(!xfr->task_nextprobe->timer) {
6863 		xfr->task_nextprobe->timer = comm_timer_create(
6864 			env->worker_base, auth_xfer_timer, xfr);
6865 		if(!xfr->task_nextprobe->timer) {
6866 			/* failed to malloc memory. likely zone transfer
6867 			 * also fails for that. skip the timeout */
6868 			char zname[255+1];
6869 			dname_str(xfr->name, zname);
6870 			log_err("cannot allocate timer, no refresh for %s",
6871 				zname);
6872 			return;
6873 		}
6874 	}
6875 	xfr->task_nextprobe->worker = env->worker;
6876 	xfr->task_nextprobe->env = env;
6877 	if(*(xfr->task_nextprobe->env->now) <= xfr->task_nextprobe->next_probe)
6878 		tv.tv_sec = xfr->task_nextprobe->next_probe -
6879 			*(xfr->task_nextprobe->env->now);
6880 	else	tv.tv_sec = 0;
6881 	if(tv.tv_sec != 0 && lookup_only && xfr->task_probe->masters) {
6882 		/* don't lookup_only, if lookup timeout is 0 anyway,
6883 		 * or if we don't have masters to lookup */
6884 		tv.tv_sec = 0;
6885 		if(xfr->task_probe->worker == NULL)
6886 			xfr->task_probe->only_lookup = 1;
6887 	}
6888 	if(verbosity >= VERB_ALGO) {
6889 		char zname[255+1];
6890 		dname_str(xfr->name, zname);
6891 		verbose(VERB_ALGO, "auth zone %s timeout in %d seconds",
6892 			zname, (int)tv.tv_sec);
6893 	}
6894 	tv.tv_usec = 0;
6895 	comm_timer_set(xfr->task_nextprobe->timer, &tv);
6896 }
6897 
6898 /** initial pick up of worker timeouts, ties events to worker event loop */
6899 void
6900 auth_xfer_pickup_initial(struct auth_zones* az, struct module_env* env)
6901 {
6902 	struct auth_xfer* x;
6903 	lock_rw_wrlock(&az->lock);
6904 	RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6905 		lock_basic_lock(&x->lock);
6906 		/* set lease_time, because we now have timestamp in env,
6907 		 * (not earlier during startup and apply_cfg), and this
6908 		 * notes the start time when the data was acquired */
6909 		if(x->have_zone)
6910 			x->lease_time = *env->now;
6911 		if(x->task_nextprobe && x->task_nextprobe->worker == NULL) {
6912 			xfr_set_timeout(x, env, 0, 1);
6913 		}
6914 		lock_basic_unlock(&x->lock);
6915 	}
6916 	lock_rw_unlock(&az->lock);
6917 }
6918 
6919 void auth_zones_cleanup(struct auth_zones* az)
6920 {
6921 	struct auth_xfer* x;
6922 	lock_rw_wrlock(&az->lock);
6923 	RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6924 		lock_basic_lock(&x->lock);
6925 		if(x->task_nextprobe && x->task_nextprobe->worker != NULL) {
6926 			xfr_nextprobe_disown(x);
6927 		}
6928 		if(x->task_probe && x->task_probe->worker != NULL) {
6929 			xfr_probe_disown(x);
6930 		}
6931 		if(x->task_transfer && x->task_transfer->worker != NULL) {
6932 			auth_chunks_delete(x->task_transfer);
6933 			xfr_transfer_disown(x);
6934 		}
6935 		lock_basic_unlock(&x->lock);
6936 	}
6937 	lock_rw_unlock(&az->lock);
6938 }
6939 
6940 /**
6941  * malloc the xfer and tasks
6942  * @param z: auth_zone with name of zone.
6943  */
6944 static struct auth_xfer*
6945 auth_xfer_new(struct auth_zone* z)
6946 {
6947 	struct auth_xfer* xfr;
6948 	xfr = (struct auth_xfer*)calloc(1, sizeof(*xfr));
6949 	if(!xfr) return NULL;
6950 	xfr->name = memdup(z->name, z->namelen);
6951 	if(!xfr->name) {
6952 		free(xfr);
6953 		return NULL;
6954 	}
6955 	xfr->node.key = xfr;
6956 	xfr->namelen = z->namelen;
6957 	xfr->namelabs = z->namelabs;
6958 	xfr->dclass = z->dclass;
6959 
6960 	xfr->task_nextprobe = (struct auth_nextprobe*)calloc(1,
6961 		sizeof(struct auth_nextprobe));
6962 	if(!xfr->task_nextprobe) {
6963 		free(xfr->name);
6964 		free(xfr);
6965 		return NULL;
6966 	}
6967 	xfr->task_probe = (struct auth_probe*)calloc(1,
6968 		sizeof(struct auth_probe));
6969 	if(!xfr->task_probe) {
6970 		free(xfr->task_nextprobe);
6971 		free(xfr->name);
6972 		free(xfr);
6973 		return NULL;
6974 	}
6975 	xfr->task_transfer = (struct auth_transfer*)calloc(1,
6976 		sizeof(struct auth_transfer));
6977 	if(!xfr->task_transfer) {
6978 		free(xfr->task_probe);
6979 		free(xfr->task_nextprobe);
6980 		free(xfr->name);
6981 		free(xfr);
6982 		return NULL;
6983 	}
6984 
6985 	lock_basic_init(&xfr->lock);
6986 	lock_protect(&xfr->lock, &xfr->name, sizeof(xfr->name));
6987 	lock_protect(&xfr->lock, &xfr->namelen, sizeof(xfr->namelen));
6988 	lock_protect(&xfr->lock, xfr->name, xfr->namelen);
6989 	lock_protect(&xfr->lock, &xfr->namelabs, sizeof(xfr->namelabs));
6990 	lock_protect(&xfr->lock, &xfr->dclass, sizeof(xfr->dclass));
6991 	lock_protect(&xfr->lock, &xfr->notify_received, sizeof(xfr->notify_received));
6992 	lock_protect(&xfr->lock, &xfr->notify_serial, sizeof(xfr->notify_serial));
6993 	lock_protect(&xfr->lock, &xfr->zone_expired, sizeof(xfr->zone_expired));
6994 	lock_protect(&xfr->lock, &xfr->have_zone, sizeof(xfr->have_zone));
6995 	lock_protect(&xfr->lock, &xfr->serial, sizeof(xfr->serial));
6996 	lock_protect(&xfr->lock, &xfr->retry, sizeof(xfr->retry));
6997 	lock_protect(&xfr->lock, &xfr->refresh, sizeof(xfr->refresh));
6998 	lock_protect(&xfr->lock, &xfr->expiry, sizeof(xfr->expiry));
6999 	lock_protect(&xfr->lock, &xfr->lease_time, sizeof(xfr->lease_time));
7000 	lock_protect(&xfr->lock, &xfr->task_nextprobe->worker,
7001 		sizeof(xfr->task_nextprobe->worker));
7002 	lock_protect(&xfr->lock, &xfr->task_probe->worker,
7003 		sizeof(xfr->task_probe->worker));
7004 	lock_protect(&xfr->lock, &xfr->task_transfer->worker,
7005 		sizeof(xfr->task_transfer->worker));
7006 	lock_basic_lock(&xfr->lock);
7007 	return xfr;
7008 }
7009 
7010 /** Create auth_xfer structure.
7011  * This populates the have_zone, soa values, and so on times.
7012  * and sets the timeout, if a zone transfer is needed a short timeout is set.
7013  * For that the auth_zone itself must exist (and read in zonefile)
7014  * returns false on alloc failure. */
7015 struct auth_xfer*
7016 auth_xfer_create(struct auth_zones* az, struct auth_zone* z)
7017 {
7018 	struct auth_xfer* xfr;
7019 
7020 	/* malloc it */
7021 	xfr = auth_xfer_new(z);
7022 	if(!xfr) {
7023 		log_err("malloc failure");
7024 		return NULL;
7025 	}
7026 	/* insert in tree */
7027 	(void)rbtree_insert(&az->xtree, &xfr->node);
7028 	return xfr;
7029 }
7030 
7031 /** create new auth_master structure */
7032 static struct auth_master*
7033 auth_master_new(struct auth_master*** list)
7034 {
7035 	struct auth_master *m;
7036 	m = (struct auth_master*)calloc(1, sizeof(*m));
7037 	if(!m) {
7038 		log_err("malloc failure");
7039 		return NULL;
7040 	}
7041 	/* set first pointer to m, or next pointer of previous element to m */
7042 	(**list) = m;
7043 	/* store m's next pointer as future point to store at */
7044 	(*list) = &(m->next);
7045 	return m;
7046 }
7047 
7048 /** dup_prefix : create string from initial part of other string, malloced */
7049 static char*
7050 dup_prefix(char* str, size_t num)
7051 {
7052 	char* result;
7053 	size_t len = strlen(str);
7054 	if(len < num) num = len; /* not more than strlen */
7055 	result = (char*)malloc(num+1);
7056 	if(!result) {
7057 		log_err("malloc failure");
7058 		return result;
7059 	}
7060 	memmove(result, str, num);
7061 	result[num] = 0;
7062 	return result;
7063 }
7064 
7065 /** dup string and print error on error */
7066 static char*
7067 dup_all(char* str)
7068 {
7069 	char* result = strdup(str);
7070 	if(!result) {
7071 		log_err("malloc failure");
7072 		return NULL;
7073 	}
7074 	return result;
7075 }
7076 
7077 /** find first of two characters */
7078 static char*
7079 str_find_first_of_chars(char* s, char a, char b)
7080 {
7081 	char* ra = strchr(s, a);
7082 	char* rb = strchr(s, b);
7083 	if(!ra) return rb;
7084 	if(!rb) return ra;
7085 	if(ra < rb) return ra;
7086 	return rb;
7087 }
7088 
7089 /** parse URL into host and file parts, false on malloc or parse error */
7090 static int
7091 parse_url(char* url, char** host, char** file, int* port, int* ssl)
7092 {
7093 	char* p = url;
7094 	/* parse http://www.example.com/file.htm
7095 	 * or http://127.0.0.1   (index.html)
7096 	 * or https://[::1@1234]/a/b/c/d */
7097 	*ssl = 1;
7098 	*port = AUTH_HTTPS_PORT;
7099 
7100 	/* parse http:// or https:// */
7101 	if(strncmp(p, "http://", 7) == 0) {
7102 		p += 7;
7103 		*ssl = 0;
7104 		*port = AUTH_HTTP_PORT;
7105 	} else if(strncmp(p, "https://", 8) == 0) {
7106 		p += 8;
7107 	} else if(strstr(p, "://") && strchr(p, '/') > strstr(p, "://") &&
7108 		strchr(p, ':') >= strstr(p, "://")) {
7109 		char* uri = dup_prefix(p, (size_t)(strstr(p, "://")-p));
7110 		log_err("protocol %s:// not supported (for url %s)",
7111 			uri?uri:"", p);
7112 		free(uri);
7113 		return 0;
7114 	}
7115 
7116 	/* parse hostname part */
7117 	if(p[0] == '[') {
7118 		char* end = strchr(p, ']');
7119 		p++; /* skip over [ */
7120 		if(end) {
7121 			*host = dup_prefix(p, (size_t)(end-p));
7122 			if(!*host) return 0;
7123 			p = end+1; /* skip over ] */
7124 		} else {
7125 			*host = dup_all(p);
7126 			if(!*host) return 0;
7127 			p = end;
7128 		}
7129 	} else {
7130 		char* end = str_find_first_of_chars(p, ':', '/');
7131 		if(end) {
7132 			*host = dup_prefix(p, (size_t)(end-p));
7133 			if(!*host) return 0;
7134 		} else {
7135 			*host = dup_all(p);
7136 			if(!*host) return 0;
7137 		}
7138 		p = end; /* at next : or / or NULL */
7139 	}
7140 
7141 	/* parse port number */
7142 	if(p && p[0] == ':') {
7143 		char* end = NULL;
7144 		*port = strtol(p+1, &end, 10);
7145 		p = end;
7146 	}
7147 
7148 	/* parse filename part */
7149 	while(p && *p == '/')
7150 		p++;
7151 	if(!p || p[0] == 0)
7152 		*file = strdup("index.html");
7153 	else	*file = strdup(p);
7154 	if(!*file) {
7155 		log_err("malloc failure");
7156 		return 0;
7157 	}
7158 	return 1;
7159 }
7160 
7161 int
7162 xfer_set_masters(struct auth_master** list, struct config_auth* c,
7163 	int with_http)
7164 {
7165 	struct auth_master* m;
7166 	struct config_strlist* p;
7167 	/* list points to the first, or next pointer for the new element */
7168 	while(*list) {
7169 		list = &( (*list)->next );
7170 	}
7171 	if(with_http)
7172 	  for(p = c->urls; p; p = p->next) {
7173 		m = auth_master_new(&list);
7174 		if(!m) return 0;
7175 		m->http = 1;
7176 		if(!parse_url(p->str, &m->host, &m->file, &m->port, &m->ssl))
7177 			return 0;
7178 	}
7179 	for(p = c->masters; p; p = p->next) {
7180 		m = auth_master_new(&list);
7181 		if(!m) return 0;
7182 		m->ixfr = 1; /* this flag is not configurable */
7183 		m->host = strdup(p->str);
7184 		if(!m->host) {
7185 			log_err("malloc failure");
7186 			return 0;
7187 		}
7188 	}
7189 	for(p = c->allow_notify; p; p = p->next) {
7190 		m = auth_master_new(&list);
7191 		if(!m) return 0;
7192 		m->allow_notify = 1;
7193 		m->host = strdup(p->str);
7194 		if(!m->host) {
7195 			log_err("malloc failure");
7196 			return 0;
7197 		}
7198 	}
7199 	return 1;
7200 }
7201 
7202 #define SERIAL_BITS	32
7203 int
7204 compare_serial(uint32_t a, uint32_t b)
7205 {
7206 	const uint32_t cutoff = ((uint32_t) 1 << (SERIAL_BITS - 1));
7207 
7208 	if (a == b) {
7209 		return 0;
7210 	} else if ((a < b && b - a < cutoff) || (a > b && a - b > cutoff)) {
7211 		return -1;
7212 	} else {
7213 		return 1;
7214 	}
7215 }
7216 
7217 int zonemd_hashalgo_supported(int hashalgo)
7218 {
7219 	if(hashalgo == ZONEMD_ALGO_SHA384) return 1;
7220 	if(hashalgo == ZONEMD_ALGO_SHA512) return 1;
7221 	return 0;
7222 }
7223 
7224 int zonemd_scheme_supported(int scheme)
7225 {
7226 	if(scheme == ZONEMD_SCHEME_SIMPLE) return 1;
7227 	return 0;
7228 }
7229 
7230 /** initialize hash for hashing with zonemd hash algo */
7231 static struct secalgo_hash* zonemd_digest_init(int hashalgo, char** reason)
7232 {
7233 	struct secalgo_hash *h;
7234 	if(hashalgo == ZONEMD_ALGO_SHA384) {
7235 		/* sha384 */
7236 		h = secalgo_hash_create_sha384();
7237 		if(!h)
7238 			*reason = "digest sha384 could not be created";
7239 		return h;
7240 	} else if(hashalgo == ZONEMD_ALGO_SHA512) {
7241 		/* sha512 */
7242 		h = secalgo_hash_create_sha512();
7243 		if(!h)
7244 			*reason = "digest sha512 could not be created";
7245 		return h;
7246 	}
7247 	/* unknown hash algo */
7248 	*reason = "unsupported algorithm";
7249 	return NULL;
7250 }
7251 
7252 /** update the hash for zonemd */
7253 static int zonemd_digest_update(int hashalgo, struct secalgo_hash* h,
7254 	uint8_t* data, size_t len, char** reason)
7255 {
7256 	if(hashalgo == ZONEMD_ALGO_SHA384) {
7257 		if(!secalgo_hash_update(h, data, len)) {
7258 			*reason = "digest sha384 failed";
7259 			return 0;
7260 		}
7261 		return 1;
7262 	} else if(hashalgo == ZONEMD_ALGO_SHA512) {
7263 		if(!secalgo_hash_update(h, data, len)) {
7264 			*reason = "digest sha512 failed";
7265 			return 0;
7266 		}
7267 		return 1;
7268 	}
7269 	/* unknown hash algo */
7270 	*reason = "unsupported algorithm";
7271 	return 0;
7272 }
7273 
7274 /** finish the hash for zonemd */
7275 static int zonemd_digest_finish(int hashalgo, struct secalgo_hash* h,
7276 	uint8_t* result, size_t hashlen, size_t* resultlen, char** reason)
7277 {
7278 	if(hashalgo == ZONEMD_ALGO_SHA384) {
7279 		if(hashlen < 384/8) {
7280 			*reason = "digest buffer too small for sha384";
7281 			return 0;
7282 		}
7283 		if(!secalgo_hash_final(h, result, hashlen, resultlen)) {
7284 			*reason = "digest sha384 finish failed";
7285 			return 0;
7286 		}
7287 		return 1;
7288 	} else if(hashalgo == ZONEMD_ALGO_SHA512) {
7289 		if(hashlen < 512/8) {
7290 			*reason = "digest buffer too small for sha512";
7291 			return 0;
7292 		}
7293 		if(!secalgo_hash_final(h, result, hashlen, resultlen)) {
7294 			*reason = "digest sha512 finish failed";
7295 			return 0;
7296 		}
7297 		return 1;
7298 	}
7299 	/* unknown algo */
7300 	*reason = "unsupported algorithm";
7301 	return 0;
7302 }
7303 
7304 /** add rrsets from node to the list */
7305 static size_t authdata_rrsets_to_list(struct auth_rrset** array,
7306 	size_t arraysize, struct auth_rrset* first)
7307 {
7308 	struct auth_rrset* rrset = first;
7309 	size_t num = 0;
7310 	while(rrset) {
7311 		if(num >= arraysize)
7312 			return num;
7313 		array[num] = rrset;
7314 		num++;
7315 		rrset = rrset->next;
7316 	}
7317 	return num;
7318 }
7319 
7320 /** compare rr list entries */
7321 static int rrlist_compare(const void* arg1, const void* arg2)
7322 {
7323 	struct auth_rrset* r1 = *(struct auth_rrset**)arg1;
7324 	struct auth_rrset* r2 = *(struct auth_rrset**)arg2;
7325 	uint16_t t1, t2;
7326 	if(r1 == NULL) t1 = LDNS_RR_TYPE_RRSIG;
7327 	else t1 = r1->type;
7328 	if(r2 == NULL) t2 = LDNS_RR_TYPE_RRSIG;
7329 	else t2 = r2->type;
7330 	if(t1 < t2)
7331 		return -1;
7332 	if(t1 > t2)
7333 		return 1;
7334 	return 0;
7335 }
7336 
7337 /** add type RRSIG to rr list if not one there already,
7338  * this is to perform RRSIG collate processing at that point. */
7339 static void addrrsigtype_if_needed(struct auth_rrset** array,
7340 	size_t arraysize, size_t* rrnum, struct auth_data* node)
7341 {
7342 	if(az_domain_rrset(node, LDNS_RR_TYPE_RRSIG))
7343 		return; /* already one there */
7344 	if((*rrnum) >= arraysize)
7345 		return; /* array too small? */
7346 	array[*rrnum] = NULL; /* nothing there, but need entry in list */
7347 	(*rrnum)++;
7348 }
7349 
7350 /** collate the RRs in an RRset using the simple scheme */
7351 static int zonemd_simple_rrset(struct auth_zone* z, int hashalgo,
7352 	struct secalgo_hash* h, struct auth_data* node,
7353 	struct auth_rrset* rrset, struct regional* region,
7354 	struct sldns_buffer* buf, char** reason)
7355 {
7356 	/* canonicalize */
7357 	struct ub_packed_rrset_key key;
7358 	memset(&key, 0, sizeof(key));
7359 	key.entry.key = &key;
7360 	key.entry.data = rrset->data;
7361 	key.rk.dname = node->name;
7362 	key.rk.dname_len = node->namelen;
7363 	key.rk.type = htons(rrset->type);
7364 	key.rk.rrset_class = htons(z->dclass);
7365 	if(!rrset_canonicalize_to_buffer(region, buf, &key)) {
7366 		*reason = "out of memory";
7367 		return 0;
7368 	}
7369 	regional_free_all(region);
7370 
7371 	/* hash */
7372 	if(!zonemd_digest_update(hashalgo, h, sldns_buffer_begin(buf),
7373 		sldns_buffer_limit(buf), reason)) {
7374 		return 0;
7375 	}
7376 	return 1;
7377 }
7378 
7379 /** count number of RRSIGs in a domain name rrset list */
7380 static size_t zonemd_simple_count_rrsig(struct auth_rrset* rrset,
7381 	struct auth_rrset** rrlist, size_t rrnum,
7382 	struct auth_zone* z, struct auth_data* node)
7383 {
7384 	size_t i, count = 0;
7385 	if(rrset) {
7386 		size_t j;
7387 		for(j = 0; j<rrset->data->count; j++) {
7388 			if(rrsig_rdata_get_type_covered(rrset->data->
7389 				rr_data[j], rrset->data->rr_len[j]) ==
7390 				LDNS_RR_TYPE_ZONEMD &&
7391 				query_dname_compare(z->name, node->name)==0) {
7392 				/* omit RRSIGs over type ZONEMD at apex */
7393 				continue;
7394 			}
7395 			count++;
7396 		}
7397 	}
7398 	for(i=0; i<rrnum; i++) {
7399 		if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7400 			query_dname_compare(z->name, node->name)==0) {
7401 			/* omit RRSIGs over type ZONEMD at apex */
7402 			continue;
7403 		}
7404 		count += (rrlist[i]?rrlist[i]->data->rrsig_count:0);
7405 	}
7406 	return count;
7407 }
7408 
7409 /** allocate sparse rrset data for the number of entries in tepm region */
7410 static int zonemd_simple_rrsig_allocs(struct regional* region,
7411 	struct packed_rrset_data* data, size_t count)
7412 {
7413 	data->rr_len = regional_alloc(region, sizeof(*data->rr_len) * count);
7414 	if(!data->rr_len) {
7415 		return 0;
7416 	}
7417 	data->rr_ttl = regional_alloc(region, sizeof(*data->rr_ttl) * count);
7418 	if(!data->rr_ttl) {
7419 		return 0;
7420 	}
7421 	data->rr_data = regional_alloc(region, sizeof(*data->rr_data) * count);
7422 	if(!data->rr_data) {
7423 		return 0;
7424 	}
7425 	return 1;
7426 }
7427 
7428 /** add the RRSIGs from the rrs in the domain into the data */
7429 static void add_rrlist_rrsigs_into_data(struct packed_rrset_data* data,
7430 	size_t* done, struct auth_rrset** rrlist, size_t rrnum,
7431 	struct auth_zone* z, struct auth_data* node)
7432 {
7433 	size_t i;
7434 	for(i=0; i<rrnum; i++) {
7435 		size_t j;
7436 		if(!rrlist[i])
7437 			continue;
7438 		if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7439 			query_dname_compare(z->name, node->name)==0) {
7440 			/* omit RRSIGs over type ZONEMD at apex */
7441 			continue;
7442 		}
7443 		for(j = 0; j<rrlist[i]->data->rrsig_count; j++) {
7444 			data->rr_len[*done] = rrlist[i]->data->rr_len[rrlist[i]->data->count + j];
7445 			data->rr_ttl[*done] = rrlist[i]->data->rr_ttl[rrlist[i]->data->count + j];
7446 			/* reference the rdata in the rrset, no need to
7447 			 * copy it, it is no longer needed at the end of
7448 			 * the routine */
7449 			data->rr_data[*done] = rrlist[i]->data->rr_data[rrlist[i]->data->count + j];
7450 			(*done)++;
7451 		}
7452 	}
7453 }
7454 
7455 static void add_rrset_into_data(struct packed_rrset_data* data,
7456 	size_t* done, struct auth_rrset* rrset,
7457 	struct auth_zone* z, struct auth_data* node)
7458 {
7459 	if(rrset) {
7460 		size_t j;
7461 		for(j = 0; j<rrset->data->count; j++) {
7462 			if(rrsig_rdata_get_type_covered(rrset->data->
7463 				rr_data[j], rrset->data->rr_len[j]) ==
7464 				LDNS_RR_TYPE_ZONEMD &&
7465 				query_dname_compare(z->name, node->name)==0) {
7466 				/* omit RRSIGs over type ZONEMD at apex */
7467 				continue;
7468 			}
7469 			data->rr_len[*done] = rrset->data->rr_len[j];
7470 			data->rr_ttl[*done] = rrset->data->rr_ttl[j];
7471 			/* reference the rdata in the rrset, no need to
7472 			 * copy it, it is no longer need at the end of
7473 			 * the routine */
7474 			data->rr_data[*done] = rrset->data->rr_data[j];
7475 			(*done)++;
7476 		}
7477 	}
7478 }
7479 
7480 /** collate the RRSIGs using the simple scheme */
7481 static int zonemd_simple_rrsig(struct auth_zone* z, int hashalgo,
7482 	struct secalgo_hash* h, struct auth_data* node,
7483 	struct auth_rrset* rrset, struct auth_rrset** rrlist, size_t rrnum,
7484 	struct regional* region, struct sldns_buffer* buf, char** reason)
7485 {
7486 	/* the rrset pointer can be NULL, this means it is type RRSIG and
7487 	 * there is no ordinary type RRSIG there.  The RRSIGs are stored
7488 	 * with the RRsets in their data.
7489 	 *
7490 	 * The RRset pointer can be nonNULL. This happens if there is
7491 	 * no RR that is covered by the RRSIG for the domain.  Then this
7492 	 * RRSIG RR is stored in an rrset of type RRSIG. The other RRSIGs
7493 	 * are stored in the rrset entries for the RRs in the rr list for
7494 	 * the domain node.  We need to collate the rrset's data, if any, and
7495 	 * the rrlist's rrsigs */
7496 	/* if this is the apex, omit RRSIGs that cover type ZONEMD */
7497 	/* build rrsig rrset */
7498 	size_t done = 0;
7499 	struct ub_packed_rrset_key key;
7500 	struct packed_rrset_data data;
7501 	memset(&key, 0, sizeof(key));
7502 	memset(&data, 0, sizeof(data));
7503 	key.entry.key = &key;
7504 	key.entry.data = &data;
7505 	key.rk.dname = node->name;
7506 	key.rk.dname_len = node->namelen;
7507 	key.rk.type = htons(LDNS_RR_TYPE_RRSIG);
7508 	key.rk.rrset_class = htons(z->dclass);
7509 	data.count = zonemd_simple_count_rrsig(rrset, rrlist, rrnum, z, node);
7510 	if(!zonemd_simple_rrsig_allocs(region, &data, data.count)) {
7511 		*reason = "out of memory";
7512 		regional_free_all(region);
7513 		return 0;
7514 	}
7515 	/* all the RRSIGs stored in the other rrsets for this domain node */
7516 	add_rrlist_rrsigs_into_data(&data, &done, rrlist, rrnum, z, node);
7517 	/* plus the RRSIGs stored in an rrset of type RRSIG for this node */
7518 	add_rrset_into_data(&data, &done, rrset, z, node);
7519 
7520 	/* canonicalize */
7521 	if(!rrset_canonicalize_to_buffer(region, buf, &key)) {
7522 		*reason = "out of memory";
7523 		regional_free_all(region);
7524 		return 0;
7525 	}
7526 	regional_free_all(region);
7527 
7528 	/* hash */
7529 	if(!zonemd_digest_update(hashalgo, h, sldns_buffer_begin(buf),
7530 		sldns_buffer_limit(buf), reason)) {
7531 		return 0;
7532 	}
7533 	return 1;
7534 }
7535 
7536 /** collate a domain's rrsets using the simple scheme */
7537 static int zonemd_simple_domain(struct auth_zone* z, int hashalgo,
7538 	struct secalgo_hash* h, struct auth_data* node,
7539 	struct regional* region, struct sldns_buffer* buf, char** reason)
7540 {
7541 	const size_t rrlistsize = 65536;
7542 	struct auth_rrset* rrlist[rrlistsize];
7543 	size_t i, rrnum = 0;
7544 	/* see if the domain is out of scope, the zone origin,
7545 	 * that would be omitted */
7546 	if(!dname_subdomain_c(node->name, z->name))
7547 		return 1; /* continue */
7548 	/* loop over the rrsets in ascending order. */
7549 	rrnum = authdata_rrsets_to_list(rrlist, rrlistsize, node->rrsets);
7550 	addrrsigtype_if_needed(rrlist, rrlistsize, &rrnum, node);
7551 	qsort(rrlist, rrnum, sizeof(*rrlist), rrlist_compare);
7552 	for(i=0; i<rrnum; i++) {
7553 		if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7554 			query_dname_compare(z->name, node->name) == 0) {
7555 			/* omit type ZONEMD at apex */
7556 			continue;
7557 		}
7558 		if(rrlist[i] == NULL || rrlist[i]->type ==
7559 			LDNS_RR_TYPE_RRSIG) {
7560 			if(!zonemd_simple_rrsig(z, hashalgo, h, node,
7561 				rrlist[i], rrlist, rrnum, region, buf, reason))
7562 				return 0;
7563 		} else if(!zonemd_simple_rrset(z, hashalgo, h, node,
7564 			rrlist[i], region, buf, reason)) {
7565 			return 0;
7566 		}
7567 	}
7568 	return 1;
7569 }
7570 
7571 /** collate the zone using the simple scheme */
7572 static int zonemd_simple_collate(struct auth_zone* z, int hashalgo,
7573 	struct secalgo_hash* h, struct regional* region,
7574 	struct sldns_buffer* buf, char** reason)
7575 {
7576 	/* our tree is sorted in canonical order, so we can just loop over
7577 	 * the tree */
7578 	struct auth_data* n;
7579 	RBTREE_FOR(n, struct auth_data*, &z->data) {
7580 		if(!zonemd_simple_domain(z, hashalgo, h, n, region, buf,
7581 			reason))
7582 			return 0;
7583 	}
7584 	return 1;
7585 }
7586 
7587 int auth_zone_generate_zonemd_hash(struct auth_zone* z, int scheme,
7588 	int hashalgo, uint8_t* hash, size_t hashlen, size_t* resultlen,
7589 	struct regional* region, struct sldns_buffer* buf, char** reason)
7590 {
7591 	struct secalgo_hash* h = zonemd_digest_init(hashalgo, reason);
7592 	if(!h) {
7593 		if(!*reason)
7594 			*reason = "digest init fail";
7595 		return 0;
7596 	}
7597 	if(scheme == ZONEMD_SCHEME_SIMPLE) {
7598 		if(!zonemd_simple_collate(z, hashalgo, h, region, buf, reason)) {
7599 			if(!*reason) *reason = "scheme simple collate fail";
7600 			secalgo_hash_delete(h);
7601 			return 0;
7602 		}
7603 	}
7604 	if(!zonemd_digest_finish(hashalgo, h, hash, hashlen, resultlen,
7605 		reason)) {
7606 		secalgo_hash_delete(h);
7607 		*reason = "digest finish fail";
7608 		return 0;
7609 	}
7610 	secalgo_hash_delete(h);
7611 	return 1;
7612 }
7613 
7614 int auth_zone_generate_zonemd_check(struct auth_zone* z, int scheme,
7615 	int hashalgo, uint8_t* hash, size_t hashlen, struct regional* region,
7616 	struct sldns_buffer* buf, char** reason)
7617 {
7618 	uint8_t gen[512];
7619 	size_t genlen = 0;
7620 	if(!zonemd_hashalgo_supported(hashalgo)) {
7621 		*reason = "unsupported algorithm";
7622 		return 0;
7623 	}
7624 	if(!zonemd_scheme_supported(scheme)) {
7625 		*reason = "unsupported scheme";
7626 		return 0;
7627 	}
7628 	if(hashlen < 12) {
7629 		/* the ZONEMD draft requires digests to fail if too small */
7630 		*reason = "digest length too small, less than 12";
7631 		return 0;
7632 	}
7633 	/* generate digest */
7634 	if(!auth_zone_generate_zonemd_hash(z, scheme, hashalgo, gen,
7635 		sizeof(gen), &genlen, region, buf, reason)) {
7636 		/* reason filled in by zonemd hash routine */
7637 		return 0;
7638 	}
7639 	/* check digest length */
7640 	if(hashlen != genlen) {
7641 		*reason = "incorrect digest length";
7642 		if(verbosity >= VERB_ALGO) {
7643 			verbose(VERB_ALGO, "zonemd scheme=%d hashalgo=%d",
7644 				scheme, hashalgo);
7645 			log_hex("ZONEMD should be  ", gen, genlen);
7646 			log_hex("ZONEMD to check is", hash, hashlen);
7647 		}
7648 		return 0;
7649 	}
7650 	/* check digest */
7651 	if(memcmp(hash, gen, genlen) != 0) {
7652 		*reason = "incorrect digest";
7653 		if(verbosity >= VERB_ALGO) {
7654 			verbose(VERB_ALGO, "zonemd scheme=%d hashalgo=%d",
7655 				scheme, hashalgo);
7656 			log_hex("ZONEMD should be  ", gen, genlen);
7657 			log_hex("ZONEMD to check is", hash, hashlen);
7658 		}
7659 		return 0;
7660 	}
7661 	return 1;
7662 }
7663 
7664 /** log auth zone message with zone name in front. */
7665 static void auth_zone_log(uint8_t* name, enum verbosity_value level,
7666 	const char* format, ...) ATTR_FORMAT(printf, 3, 4);
7667 static void auth_zone_log(uint8_t* name, enum verbosity_value level,
7668 	const char* format, ...)
7669 {
7670 	va_list args;
7671 	va_start(args, format);
7672 	if(verbosity >= level) {
7673 		char str[255+1];
7674 		char msg[MAXSYSLOGMSGLEN];
7675 		dname_str(name, str);
7676 		vsnprintf(msg, sizeof(msg), format, args);
7677 		verbose(level, "auth zone %s %s", str, msg);
7678 	}
7679 	va_end(args);
7680 }
7681 
7682 /** ZONEMD, dnssec verify the rrset with the dnskey */
7683 static int zonemd_dnssec_verify_rrset(struct auth_zone* z,
7684 	struct module_env* env, struct module_stack* mods,
7685 	struct ub_packed_rrset_key* dnskey, struct auth_data* node,
7686 	struct auth_rrset* rrset, char** why_bogus)
7687 {
7688 	struct ub_packed_rrset_key pk;
7689 	enum sec_status sec;
7690 	struct val_env* ve;
7691 	int m;
7692 	m = modstack_find(mods, "validator");
7693 	if(m == -1) {
7694 		auth_zone_log(z->name, VERB_ALGO, "zonemd dnssec verify: have "
7695 			"DNSKEY chain of trust, but no validator module");
7696 		return 0;
7697 	}
7698 	ve = (struct val_env*)env->modinfo[m];
7699 
7700 	memset(&pk, 0, sizeof(pk));
7701 	pk.entry.key = &pk;
7702 	pk.entry.data = rrset->data;
7703 	pk.rk.dname = node->name;
7704 	pk.rk.dname_len = node->namelen;
7705 	pk.rk.type = htons(rrset->type);
7706 	pk.rk.rrset_class = htons(z->dclass);
7707 	if(verbosity >= VERB_ALGO) {
7708 		char typestr[32];
7709 		typestr[0]=0;
7710 		sldns_wire2str_type_buf(rrset->type, typestr, sizeof(typestr));
7711 		auth_zone_log(z->name, VERB_ALGO,
7712 			"zonemd: verify %s RRset with DNSKEY", typestr);
7713 	}
7714 	sec = dnskeyset_verify_rrset(env, ve, &pk, dnskey, NULL, why_bogus,
7715 		LDNS_SECTION_ANSWER, NULL);
7716 	if(sec == sec_status_secure) {
7717 		return 1;
7718 	}
7719 	if(why_bogus)
7720 		auth_zone_log(z->name, VERB_ALGO, "DNSSEC verify was bogus: %s", *why_bogus);
7721 	return 0;
7722 }
7723 
7724 /** check for nsec3, the RR with params equal, if bitmap has the type */
7725 static int nsec3_of_param_has_type(struct auth_rrset* nsec3, int algo,
7726 	size_t iter, uint8_t* salt, size_t saltlen, uint16_t rrtype)
7727 {
7728 	int i, count = (int)nsec3->data->count;
7729 	struct ub_packed_rrset_key pk;
7730 	memset(&pk, 0, sizeof(pk));
7731 	pk.entry.data = nsec3->data;
7732 	for(i=0; i<count; i++) {
7733 		int rralgo;
7734 		size_t rriter, rrsaltlen;
7735 		uint8_t* rrsalt;
7736 		if(!nsec3_get_params(&pk, i, &rralgo, &rriter, &rrsalt,
7737 			&rrsaltlen))
7738 			continue; /* no parameters, malformed */
7739 		if(rralgo != algo || rriter != iter || rrsaltlen != saltlen)
7740 			continue; /* different parameters */
7741 		if(saltlen != 0) {
7742 			if(rrsalt == NULL || salt == NULL)
7743 				continue;
7744 			if(memcmp(rrsalt, salt, saltlen) != 0)
7745 				continue; /* different salt parameters */
7746 		}
7747 		if(nsec3_has_type(&pk, i, rrtype))
7748 			return 1;
7749 	}
7750 	return 0;
7751 }
7752 
7753 /** Verify the absence of ZONEMD with DNSSEC by checking NSEC, NSEC3 type flag.
7754  * return false on failure, reason contains description of failure. */
7755 static int zonemd_check_dnssec_absence(struct auth_zone* z,
7756 	struct module_env* env, struct module_stack* mods,
7757 	struct ub_packed_rrset_key* dnskey, struct auth_data* apex,
7758 	char** reason, char** why_bogus)
7759 {
7760 	struct auth_rrset* nsec = NULL;
7761 	if(!apex) {
7762 		*reason = "zone has no apex domain but ZONEMD missing";
7763 		return 0;
7764 	}
7765 	nsec = az_domain_rrset(apex, LDNS_RR_TYPE_NSEC);
7766 	if(nsec) {
7767 		struct ub_packed_rrset_key pk;
7768 		/* dnssec verify the NSEC */
7769 		if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex,
7770 			nsec, why_bogus)) {
7771 			*reason = "DNSSEC verify failed for NSEC RRset";
7772 			return 0;
7773 		}
7774 		/* check type bitmap */
7775 		memset(&pk, 0, sizeof(pk));
7776 		pk.entry.data = nsec->data;
7777 		if(nsec_has_type(&pk, LDNS_RR_TYPE_ZONEMD)) {
7778 			*reason = "DNSSEC NSEC bitmap says type ZONEMD exists";
7779 			return 0;
7780 		}
7781 		auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC NSEC verification of absence of ZONEMD secure");
7782 	} else {
7783 		/* NSEC3 perhaps ? */
7784 		int algo;
7785 		size_t iter, saltlen;
7786 		uint8_t* salt;
7787 		struct auth_rrset* nsec3param = az_domain_rrset(apex,
7788 			LDNS_RR_TYPE_NSEC3PARAM);
7789 		struct auth_data* match;
7790 		struct auth_rrset* nsec3;
7791 		if(!nsec3param) {
7792 			*reason = "zone has no NSEC information but ZONEMD missing";
7793 			return 0;
7794 		}
7795 		if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen)) {
7796 			*reason = "zone has no NSEC information but ZONEMD missing";
7797 			return 0;
7798 		}
7799 		/* find the NSEC3 record */
7800 		match = az_nsec3_find_exact(z, z->name, z->namelen, algo,
7801 			iter, salt, saltlen);
7802 		if(!match) {
7803 			*reason = "zone has no NSEC3 domain for the apex but ZONEMD missing";
7804 			return 0;
7805 		}
7806 		nsec3 = az_domain_rrset(match, LDNS_RR_TYPE_NSEC3);
7807 		if(!nsec3) {
7808 			*reason = "zone has no NSEC3 RRset for the apex but ZONEMD missing";
7809 			return 0;
7810 		}
7811 		/* dnssec verify the NSEC3 */
7812 		if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, match,
7813 			nsec3, why_bogus)) {
7814 			*reason = "DNSSEC verify failed for NSEC3 RRset";
7815 			return 0;
7816 		}
7817 		/* check type bitmap */
7818 		if(nsec3_of_param_has_type(nsec3, algo, iter, salt, saltlen,
7819 			LDNS_RR_TYPE_ZONEMD)) {
7820 			*reason = "DNSSEC NSEC3 bitmap says type ZONEMD exists";
7821 			return 0;
7822 		}
7823 		auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC NSEC3 verification of absence of ZONEMD secure");
7824 	}
7825 
7826 	return 1;
7827 }
7828 
7829 /** Verify the SOA and ZONEMD DNSSEC signatures.
7830  * return false on failure, reason contains description of failure. */
7831 static int zonemd_check_dnssec_soazonemd(struct auth_zone* z,
7832 	struct module_env* env, struct module_stack* mods,
7833 	struct ub_packed_rrset_key* dnskey, struct auth_data* apex,
7834 	struct auth_rrset* zonemd_rrset, char** reason, char** why_bogus)
7835 {
7836 	struct auth_rrset* soa;
7837 	if(!apex) {
7838 		*reason = "zone has no apex domain";
7839 		return 0;
7840 	}
7841 	soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
7842 	if(!soa) {
7843 		*reason = "zone has no SOA RRset";
7844 		return 0;
7845 	}
7846 	if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex, soa,
7847 		why_bogus)) {
7848 		*reason = "DNSSEC verify failed for SOA RRset";
7849 		return 0;
7850 	}
7851 	if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex,
7852 		zonemd_rrset, why_bogus)) {
7853 		*reason = "DNSSEC verify failed for ZONEMD RRset";
7854 		return 0;
7855 	}
7856 	auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC verification of SOA and ZONEMD RRsets secure");
7857 	return 1;
7858 }
7859 
7860 /**
7861  * Fail the ZONEMD verification.
7862  * @param z: auth zone that fails.
7863  * @param env: environment with config, to ignore failure or not.
7864  * @param reason: failure string description.
7865  * @param why_bogus: failure string for DNSSEC verification failure.
7866  * @param result: strdup result in here if not NULL.
7867  */
7868 static void auth_zone_zonemd_fail(struct auth_zone* z, struct module_env* env,
7869 	char* reason, char* why_bogus, char** result)
7870 {
7871 	char zstr[255+1];
7872 	/* if fail: log reason, and depending on config also take action
7873 	 * and drop the zone, eg. it is gone from memory, set zone_expired */
7874 	dname_str(z->name, zstr);
7875 	if(!reason) reason = "verification failed";
7876 	if(result) {
7877 		if(why_bogus) {
7878 			char res[1024];
7879 			snprintf(res, sizeof(res), "%s: %s", reason,
7880 				why_bogus);
7881 			*result = strdup(res);
7882 		} else {
7883 			*result = strdup(reason);
7884 		}
7885 		if(!*result) log_err("out of memory");
7886 	} else {
7887 		log_warn("auth zone %s: ZONEMD verification failed: %s", zstr, reason);
7888 	}
7889 
7890 	if(env->cfg->zonemd_permissive_mode) {
7891 		verbose(VERB_ALGO, "zonemd-permissive-mode enabled, "
7892 			"not blocking zone %s", zstr);
7893 		return;
7894 	}
7895 
7896 	/* expired means the zone gives servfail and is not used by
7897 	 * lookup if fallback_enabled*/
7898 	z->zone_expired = 1;
7899 }
7900 
7901 /**
7902  * Verify the zonemd with DNSSEC and hash check, with given key.
7903  * @param z: auth zone.
7904  * @param env: environment with config and temp buffers.
7905  * @param mods: module stack with validator env for verification.
7906  * @param dnskey: dnskey that we can use, or NULL.  If nonnull, the key
7907  * 	has been verified and is the start of the chain of trust.
7908  * @param is_insecure: if true, the dnskey is not used, the zone is insecure.
7909  * 	And dnssec is not used.  It is DNSSEC secure insecure or not under
7910  * 	a trust anchor.
7911  * @param result: if not NULL result reason copied here.
7912  */
7913 static void
7914 auth_zone_verify_zonemd_with_key(struct auth_zone* z, struct module_env* env,
7915 	struct module_stack* mods, struct ub_packed_rrset_key* dnskey,
7916 	int is_insecure, char** result)
7917 {
7918 	char* reason = NULL, *why_bogus = NULL;
7919 	struct auth_data* apex = NULL;
7920 	struct auth_rrset* zonemd_rrset = NULL;
7921 	int zonemd_absent = 0, zonemd_absence_dnssecok = 0;
7922 
7923 	/* see if ZONEMD is present or absent. */
7924 	apex = az_find_name(z, z->name, z->namelen);
7925 	if(!apex) {
7926 		zonemd_absent = 1;
7927 	} else {
7928 		zonemd_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_ZONEMD);
7929 		if(!zonemd_rrset || zonemd_rrset->data->count==0) {
7930 			zonemd_absent = 1;
7931 			zonemd_rrset = NULL;
7932 		}
7933 	}
7934 
7935 	/* if no DNSSEC, done. */
7936 	/* if no ZONEMD, and DNSSEC, use DNSKEY to verify NSEC or NSEC3 for
7937 	 * zone apex.  Check ZONEMD bit is turned off or else fail */
7938 	/* if ZONEMD, and DNSSEC, check DNSSEC signature on SOA and ZONEMD,
7939 	 * or else fail */
7940 	if(!dnskey && !is_insecure) {
7941 		auth_zone_zonemd_fail(z, env, "DNSKEY missing", NULL, result);
7942 		return;
7943 	} else if(!zonemd_rrset && dnskey && !is_insecure) {
7944 		/* fetch, DNSSEC verify, and check NSEC/NSEC3 */
7945 		if(!zonemd_check_dnssec_absence(z, env, mods, dnskey, apex,
7946 			&reason, &why_bogus)) {
7947 			auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
7948 			return;
7949 		}
7950 		zonemd_absence_dnssecok = 1;
7951 	} else if(zonemd_rrset && dnskey && !is_insecure) {
7952 		/* check DNSSEC verify of SOA and ZONEMD */
7953 		if(!zonemd_check_dnssec_soazonemd(z, env, mods, dnskey, apex,
7954 			zonemd_rrset, &reason, &why_bogus)) {
7955 			auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
7956 			return;
7957 		}
7958 	}
7959 
7960 	if(zonemd_absent && z->zonemd_reject_absence) {
7961 		auth_zone_zonemd_fail(z, env, "ZONEMD absent and that is not allowed by config", NULL, result);
7962 		return;
7963 	}
7964 	if(zonemd_absent && zonemd_absence_dnssecok) {
7965 		auth_zone_log(z->name, VERB_ALGO, "DNSSEC verified nonexistence of ZONEMD");
7966 		if(result) {
7967 			*result = strdup("DNSSEC verified nonexistence of ZONEMD");
7968 			if(!*result) log_err("out of memory");
7969 		}
7970 		return;
7971 	}
7972 	if(zonemd_absent) {
7973 		auth_zone_log(z->name, VERB_ALGO, "no ZONEMD present");
7974 		if(result) {
7975 			*result = strdup("no ZONEMD present");
7976 			if(!*result) log_err("out of memory");
7977 		}
7978 		return;
7979 	}
7980 
7981 	/* check ZONEMD checksum and report or else fail. */
7982 	if(!auth_zone_zonemd_check_hash(z, env, &reason)) {
7983 		auth_zone_zonemd_fail(z, env, reason, NULL, result);
7984 		return;
7985 	}
7986 
7987 	/* success! log the success */
7988 	auth_zone_log(z->name, VERB_ALGO, "ZONEMD verification successful");
7989 	if(result) {
7990 		*result = strdup("ZONEMD verification successful");
7991 		if(!*result) log_err("out of memory");
7992 	}
7993 }
7994 
7995 /**
7996  * verify the zone DNSKEY rrset from the trust anchor
7997  * This is possible because the anchor is for the zone itself, and can
7998  * thus apply straight to the zone DNSKEY set.
7999  * @param z: the auth zone.
8000  * @param env: environment with time and temp buffers.
8001  * @param mods: module stack for validator environment for dnssec validation.
8002  * @param anchor: trust anchor to use
8003  * @param is_insecure: returned, true if the zone is securely insecure.
8004  * @param why_bogus: if the routine fails, returns the failure reason.
8005  * @param keystorage: where to store the ub_packed_rrset_key that is created
8006  * 	on success. A pointer to it is returned on success.
8007  * @return the dnskey RRset, reference to zone data and keystorage, or
8008  * 	NULL on failure.
8009  */
8010 static struct ub_packed_rrset_key*
8011 zonemd_get_dnskey_from_anchor(struct auth_zone* z, struct module_env* env,
8012 	struct module_stack* mods, struct trust_anchor* anchor,
8013 	int* is_insecure, char** why_bogus,
8014 	struct ub_packed_rrset_key* keystorage)
8015 {
8016 	struct auth_data* apex;
8017 	struct auth_rrset* dnskey_rrset;
8018 	enum sec_status sec;
8019 	struct val_env* ve;
8020 	int m;
8021 
8022 	apex = az_find_name(z, z->name, z->namelen);
8023 	if(!apex) {
8024 		*why_bogus = "have trust anchor, but zone has no apex domain for DNSKEY";
8025 		return 0;
8026 	}
8027 	dnskey_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_DNSKEY);
8028 	if(!dnskey_rrset || dnskey_rrset->data->count==0) {
8029 		*why_bogus = "have trust anchor, but zone has no DNSKEY";
8030 		return 0;
8031 	}
8032 
8033 	m = modstack_find(mods, "validator");
8034 	if(m == -1) {
8035 		*why_bogus = "have trust anchor, but no validator module";
8036 		return 0;
8037 	}
8038 	ve = (struct val_env*)env->modinfo[m];
8039 
8040 	memset(keystorage, 0, sizeof(*keystorage));
8041 	keystorage->entry.key = keystorage;
8042 	keystorage->entry.data = dnskey_rrset->data;
8043 	keystorage->rk.dname = apex->name;
8044 	keystorage->rk.dname_len = apex->namelen;
8045 	keystorage->rk.type = htons(LDNS_RR_TYPE_DNSKEY);
8046 	keystorage->rk.rrset_class = htons(z->dclass);
8047 	auth_zone_log(z->name, VERB_QUERY,
8048 		"zonemd: verify DNSKEY RRset with trust anchor");
8049 	sec = val_verify_DNSKEY_with_TA(env, ve, keystorage, anchor->ds_rrset,
8050 		anchor->dnskey_rrset, NULL, why_bogus, NULL);
8051 	regional_free_all(env->scratch);
8052 	if(sec == sec_status_secure) {
8053 		/* success */
8054 		*is_insecure = 0;
8055 		return keystorage;
8056 	} else if(sec == sec_status_insecure) {
8057 		/* insecure */
8058 		*is_insecure = 1;
8059 	} else {
8060 		/* bogus */
8061 		*is_insecure = 0;
8062 		auth_zone_log(z->name, VERB_ALGO,
8063 			"zonemd: verify DNSKEY RRset with trust anchor failed: %s", *why_bogus);
8064 	}
8065 	return NULL;
8066 }
8067 
8068 /** callback for ZONEMD lookup of DNSKEY */
8069 void auth_zonemd_dnskey_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
8070 	enum sec_status sec, char* why_bogus, int ATTR_UNUSED(was_ratelimited))
8071 {
8072 	struct auth_zone* z = (struct auth_zone*)arg;
8073 	struct module_env* env;
8074 	char* reason = NULL;
8075 	struct ub_packed_rrset_key* dnskey = NULL;
8076 	int is_insecure = 0;
8077 
8078 	lock_rw_wrlock(&z->lock);
8079 	env = z->zonemd_callback_env;
8080 	/* release the env variable so another worker can pick up the
8081 	 * ZONEMD verification task if it wants to */
8082 	z->zonemd_callback_env = NULL;
8083 	if(!env || env->outnet->want_to_quit || z->zone_deleted) {
8084 		lock_rw_unlock(&z->lock);
8085 		return; /* stop on quit */
8086 	}
8087 
8088 	/* process result */
8089 	if(sec == sec_status_bogus) {
8090 		reason = why_bogus;
8091 		if(!reason)
8092 			reason = "lookup of DNSKEY was bogus";
8093 		auth_zone_log(z->name, VERB_ALGO,
8094 			"zonemd lookup of DNSKEY was bogus: %s", reason);
8095 	} else if(rcode == LDNS_RCODE_NOERROR) {
8096 		uint16_t wanted_qtype = LDNS_RR_TYPE_DNSKEY;
8097 		struct regional* temp = env->scratch;
8098 		struct query_info rq;
8099 		struct reply_info* rep;
8100 		memset(&rq, 0, sizeof(rq));
8101 		rep = parse_reply_in_temp_region(buf, temp, &rq);
8102 		if(rep && rq.qtype == wanted_qtype &&
8103 			query_dname_compare(z->name, rq.qname) == 0 &&
8104 			FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
8105 			/* parsed successfully */
8106 			struct ub_packed_rrset_key* answer =
8107 				reply_find_answer_rrset(&rq, rep);
8108 			if(answer && sec == sec_status_secure) {
8109 				dnskey = answer;
8110 				auth_zone_log(z->name, VERB_ALGO,
8111 					"zonemd lookup of DNSKEY was secure");
8112 			} else if(sec == sec_status_secure && !answer) {
8113 				is_insecure = 1;
8114 				auth_zone_log(z->name, VERB_ALGO,
8115 					"zonemd lookup of DNSKEY has no content, but is secure, treat as insecure");
8116 			} else if(sec == sec_status_insecure) {
8117 				is_insecure = 1;
8118 				auth_zone_log(z->name, VERB_ALGO,
8119 					"zonemd lookup of DNSKEY was insecure");
8120 			} else if(sec == sec_status_indeterminate) {
8121 				is_insecure = 1;
8122 				auth_zone_log(z->name, VERB_ALGO,
8123 					"zonemd lookup of DNSKEY was indeterminate, treat as insecure");
8124 			} else {
8125 				auth_zone_log(z->name, VERB_ALGO,
8126 					"zonemd lookup of DNSKEY has nodata");
8127 				reason = "lookup of DNSKEY has nodata";
8128 			}
8129 		} else if(rep && rq.qtype == wanted_qtype &&
8130 			query_dname_compare(z->name, rq.qname) == 0 &&
8131 			FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8132 			sec == sec_status_secure) {
8133 			/* secure nxdomain, so the zone is like some RPZ zone
8134 			 * that does not exist in the wider internet, with
8135 			 * a secure nxdomain answer outside of it. So we
8136 			 * treat the zonemd zone without a dnssec chain of
8137 			 * trust, as insecure. */
8138 			is_insecure = 1;
8139 			auth_zone_log(z->name, VERB_ALGO,
8140 				"zonemd lookup of DNSKEY was secure NXDOMAIN, treat as insecure");
8141 		} else if(rep && rq.qtype == wanted_qtype &&
8142 			query_dname_compare(z->name, rq.qname) == 0 &&
8143 			FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8144 			sec == sec_status_insecure) {
8145 			is_insecure = 1;
8146 			auth_zone_log(z->name, VERB_ALGO,
8147 				"zonemd lookup of DNSKEY was insecure NXDOMAIN, treat as insecure");
8148 		} else if(rep && rq.qtype == wanted_qtype &&
8149 			query_dname_compare(z->name, rq.qname) == 0 &&
8150 			FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8151 			sec == sec_status_indeterminate) {
8152 			is_insecure = 1;
8153 			auth_zone_log(z->name, VERB_ALGO,
8154 				"zonemd lookup of DNSKEY was indeterminate NXDOMAIN, treat as insecure");
8155 		} else {
8156 			auth_zone_log(z->name, VERB_ALGO,
8157 				"zonemd lookup of DNSKEY has no answer");
8158 			reason = "lookup of DNSKEY has no answer";
8159 		}
8160 	} else {
8161 		auth_zone_log(z->name, VERB_ALGO,
8162 			"zonemd lookup of DNSKEY failed");
8163 		reason = "lookup of DNSKEY failed";
8164 	}
8165 
8166 	if(reason) {
8167 		auth_zone_zonemd_fail(z, env, reason, NULL, NULL);
8168 		lock_rw_unlock(&z->lock);
8169 		return;
8170 	}
8171 
8172 	auth_zone_verify_zonemd_with_key(z, env, &env->mesh->mods, dnskey,
8173 		is_insecure, NULL);
8174 	regional_free_all(env->scratch);
8175 	lock_rw_unlock(&z->lock);
8176 }
8177 
8178 /** lookup DNSKEY for ZONEMD verification */
8179 static int
8180 zonemd_lookup_dnskey(struct auth_zone* z, struct module_env* env)
8181 {
8182 	struct query_info qinfo;
8183 	uint16_t qflags = BIT_RD;
8184 	struct edns_data edns;
8185 	sldns_buffer* buf = env->scratch_buffer;
8186 
8187 	if(z->zonemd_callback_env) {
8188 		/* another worker is already working on the callback
8189 		 * for the DNSKEY lookup for ZONEMD verification.
8190 		 * We do not also have to do ZONEMD verification, let that
8191 		 * worker do it */
8192 		auth_zone_log(z->name, VERB_ALGO,
8193 			"zonemd needs lookup of DNSKEY and that already worked on by another worker");
8194 		return 1;
8195 	}
8196 
8197 	/* use mesh_new_callback to lookup the DNSKEY,
8198 	 * and then wait for them to be looked up (in cache, or query) */
8199 	qinfo.qname_len = z->namelen;
8200 	qinfo.qname = z->name;
8201 	qinfo.qclass = z->dclass;
8202 	qinfo.qtype = LDNS_RR_TYPE_DNSKEY;
8203 	qinfo.local_alias = NULL;
8204 	if(verbosity >= VERB_ALGO) {
8205 		char buf1[512];
8206 		char buf2[LDNS_MAX_DOMAINLEN+1];
8207 		dname_str(z->name, buf2);
8208 		snprintf(buf1, sizeof(buf1), "auth zone %s: lookup DNSKEY "
8209 			"for zonemd verification", buf2);
8210 		log_query_info(VERB_ALGO, buf1, &qinfo);
8211 	}
8212 	edns.edns_present = 1;
8213 	edns.ext_rcode = 0;
8214 	edns.edns_version = 0;
8215 	edns.bits = EDNS_DO;
8216 	edns.opt_list = NULL;
8217 	if(sldns_buffer_capacity(buf) < 65535)
8218 		edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
8219 	else	edns.udp_size = 65535;
8220 
8221 	/* store the worker-specific module env for the callback.
8222 	 * We can then reference this when the callback executes */
8223 	z->zonemd_callback_env = env;
8224 	/* the callback can be called straight away */
8225 	lock_rw_unlock(&z->lock);
8226 	if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
8227 		&auth_zonemd_dnskey_lookup_callback, z)) {
8228 		lock_rw_wrlock(&z->lock);
8229 		log_err("out of memory lookup up dnskey for zonemd");
8230 		return 0;
8231 	}
8232 	lock_rw_wrlock(&z->lock);
8233 	return 1;
8234 }
8235 
8236 void auth_zone_verify_zonemd(struct auth_zone* z, struct module_env* env,
8237 	struct module_stack* mods, char** result, int offline, int only_online)
8238 {
8239 	char* reason = NULL, *why_bogus = NULL;
8240 	struct trust_anchor* anchor = NULL;
8241 	struct ub_packed_rrset_key* dnskey = NULL;
8242 	struct ub_packed_rrset_key keystorage;
8243 	int is_insecure = 0;
8244 	/* verify the ZONEMD if present.
8245 	 * If not present check if absence is allowed by DNSSEC */
8246 	if(!z->zonemd_check)
8247 		return;
8248 
8249 	/* if zone is under a trustanchor */
8250 	/* is it equal to trustanchor - get dnskey's verified */
8251 	/* else, find chain of trust by fetching DNSKEYs lookup for zone */
8252 	/* result if that, if insecure, means no DNSSEC for the ZONEMD,
8253 	 * otherwise we have the zone DNSKEY for the DNSSEC verification. */
8254 	if(env->anchors)
8255 		anchor = anchors_lookup(env->anchors, z->name, z->namelen,
8256 			z->dclass);
8257 	if(anchor && anchor->numDS == 0 && anchor->numDNSKEY == 0) {
8258 		/* domain-insecure trust anchor for unsigned zones */
8259 		lock_basic_unlock(&anchor->lock);
8260 		if(only_online)
8261 			return;
8262 		dnskey = NULL;
8263 		is_insecure = 1;
8264 	} else if(anchor && query_dname_compare(z->name, anchor->name) == 0) {
8265 		if(only_online) {
8266 			lock_basic_unlock(&anchor->lock);
8267 			return;
8268 		}
8269 		/* equal to trustanchor, no need for online lookups */
8270 		dnskey = zonemd_get_dnskey_from_anchor(z, env, mods, anchor,
8271 			&is_insecure, &why_bogus, &keystorage);
8272 		lock_basic_unlock(&anchor->lock);
8273 		if(!dnskey && !reason && !is_insecure) {
8274 			reason = "verify DNSKEY RRset with trust anchor failed";
8275 		}
8276 	} else if(anchor) {
8277 		lock_basic_unlock(&anchor->lock);
8278 		/* perform online lookups */
8279 		if(offline)
8280 			return;
8281 		/* setup online lookups, and wait for them */
8282 		if(zonemd_lookup_dnskey(z, env)) {
8283 			/* wait for the lookup */
8284 			return;
8285 		}
8286 		reason = "could not lookup DNSKEY for chain of trust";
8287 	} else {
8288 		/* the zone is not under a trust anchor */
8289 		if(only_online)
8290 			return;
8291 		dnskey = NULL;
8292 		is_insecure = 1;
8293 	}
8294 
8295 	if(reason) {
8296 		auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
8297 		return;
8298 	}
8299 
8300 	auth_zone_verify_zonemd_with_key(z, env, mods, dnskey, is_insecure,
8301 		result);
8302 	regional_free_all(env->scratch);
8303 }
8304 
8305 void auth_zones_pickup_zonemd_verify(struct auth_zones* az,
8306 	struct module_env* env)
8307 {
8308 	struct auth_zone key;
8309 	uint8_t savezname[255+1];
8310 	size_t savezname_len;
8311 	struct auth_zone* z;
8312 	key.node.key = &key;
8313 	lock_rw_rdlock(&az->lock);
8314 	RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
8315 		lock_rw_wrlock(&z->lock);
8316 		if(!z->zonemd_check) {
8317 			lock_rw_unlock(&z->lock);
8318 			continue;
8319 		}
8320 		key.dclass = z->dclass;
8321 		key.namelabs = z->namelabs;
8322 		if(z->namelen > sizeof(savezname)) {
8323 			lock_rw_unlock(&z->lock);
8324 			log_err("auth_zones_pickup_zonemd_verify: zone name too long");
8325 			continue;
8326 		}
8327 		savezname_len = z->namelen;
8328 		memmove(savezname, z->name, z->namelen);
8329 		lock_rw_unlock(&az->lock);
8330 		auth_zone_verify_zonemd(z, env, &env->mesh->mods, NULL, 0, 1);
8331 		lock_rw_unlock(&z->lock);
8332 		lock_rw_rdlock(&az->lock);
8333 		/* find the zone we had before, it is not deleted,
8334 		 * because we have a flag for that that is processed at
8335 		 * apply_cfg time */
8336 		key.namelen = savezname_len;
8337 		key.name = savezname;
8338 		z = (struct auth_zone*)rbtree_search(&az->ztree, &key);
8339 		if(!z)
8340 			break;
8341 	}
8342 	lock_rw_unlock(&az->lock);
8343 }
8344