xref: /freebsd/contrib/unbound/services/outside_network.c (revision ab00ac327a66a53edaac95b536b209db3ae2cd9f)
1 /*
2  * services/outside_network.c - implement sending of queries and wait answer.
3  *
4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
5  *
6  * This software is open source.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright notice,
13  * this list of conditions and the following disclaimer.
14  *
15  * Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  *
19  * Neither the name of the NLNET LABS nor the names of its contributors may
20  * be used to endorse or promote products derived from this software without
21  * specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 /**
37  * \file
38  *
39  * This file has functions to send queries to authoritative servers and
40  * wait for the pending answer events.
41  */
42 #include "config.h"
43 #include <ctype.h>
44 #ifdef HAVE_SYS_TYPES_H
45 #  include <sys/types.h>
46 #endif
47 #include <sys/time.h>
48 #include "services/outside_network.h"
49 #include "services/listen_dnsport.h"
50 #include "services/cache/infra.h"
51 #include "util/data/msgparse.h"
52 #include "util/data/msgreply.h"
53 #include "util/data/msgencode.h"
54 #include "util/data/dname.h"
55 #include "util/netevent.h"
56 #include "util/log.h"
57 #include "util/net_help.h"
58 #include "util/random.h"
59 #include "util/fptr_wlist.h"
60 #include "sldns/sbuffer.h"
61 #include "dnstap/dnstap.h"
62 #ifdef HAVE_OPENSSL_SSL_H
63 #include <openssl/ssl.h>
64 #endif
65 
66 #ifdef HAVE_NETDB_H
67 #include <netdb.h>
68 #endif
69 #include <fcntl.h>
70 
71 /** number of times to retry making a random ID that is unique. */
72 #define MAX_ID_RETRY 1000
73 /** number of times to retry finding interface, port that can be opened. */
74 #define MAX_PORT_RETRY 10000
75 /** number of retries on outgoing UDP queries */
76 #define OUTBOUND_UDP_RETRY 1
77 
78 /** initiate TCP transaction for serviced query */
79 static void serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff);
80 /** with a fd available, randomize and send UDP */
81 static int randomize_and_send_udp(struct pending* pend, sldns_buffer* packet,
82 	int timeout);
83 
84 /** remove waiting tcp from the outnet waiting list */
85 static void waiting_list_remove(struct outside_network* outnet,
86 	struct waiting_tcp* w);
87 
88 int
89 pending_cmp(const void* key1, const void* key2)
90 {
91 	struct pending *p1 = (struct pending*)key1;
92 	struct pending *p2 = (struct pending*)key2;
93 	if(p1->id < p2->id)
94 		return -1;
95 	if(p1->id > p2->id)
96 		return 1;
97 	log_assert(p1->id == p2->id);
98 	return sockaddr_cmp(&p1->addr, p1->addrlen, &p2->addr, p2->addrlen);
99 }
100 
101 int
102 serviced_cmp(const void* key1, const void* key2)
103 {
104 	struct serviced_query* q1 = (struct serviced_query*)key1;
105 	struct serviced_query* q2 = (struct serviced_query*)key2;
106 	int r;
107 	if(q1->qbuflen < q2->qbuflen)
108 		return -1;
109 	if(q1->qbuflen > q2->qbuflen)
110 		return 1;
111 	log_assert(q1->qbuflen == q2->qbuflen);
112 	log_assert(q1->qbuflen >= 15 /* 10 header, root, type, class */);
113 	/* alternate casing of qname is still the same query */
114 	if((r = memcmp(q1->qbuf, q2->qbuf, 10)) != 0)
115 		return r;
116 	if((r = memcmp(q1->qbuf+q1->qbuflen-4, q2->qbuf+q2->qbuflen-4, 4)) != 0)
117 		return r;
118 	if(q1->dnssec != q2->dnssec) {
119 		if(q1->dnssec < q2->dnssec)
120 			return -1;
121 		return 1;
122 	}
123 	if((r = query_dname_compare(q1->qbuf+10, q2->qbuf+10)) != 0)
124 		return r;
125 	if((r = edns_opt_list_compare(q1->opt_list, q2->opt_list)) != 0)
126 		return r;
127 	return sockaddr_cmp(&q1->addr, q1->addrlen, &q2->addr, q2->addrlen);
128 }
129 
130 /** delete waiting_tcp entry. Does not unlink from waiting list.
131  * @param w: to delete.
132  */
133 static void
134 waiting_tcp_delete(struct waiting_tcp* w)
135 {
136 	if(!w) return;
137 	if(w->timer)
138 		comm_timer_delete(w->timer);
139 	free(w);
140 }
141 
142 /**
143  * Pick random outgoing-interface of that family, and bind it.
144  * port set to 0 so OS picks a port number for us.
145  * if it is the ANY address, do not bind.
146  * @param w: tcp structure with destination address.
147  * @param s: socket fd.
148  * @return false on error, socket closed.
149  */
150 static int
151 pick_outgoing_tcp(struct waiting_tcp* w, int s)
152 {
153 	struct port_if* pi = NULL;
154 	int num;
155 #ifdef INET6
156 	if(addr_is_ip6(&w->addr, w->addrlen))
157 		num = w->outnet->num_ip6;
158 	else
159 #endif
160 		num = w->outnet->num_ip4;
161 	if(num == 0) {
162 		log_err("no TCP outgoing interfaces of family");
163 		log_addr(VERB_OPS, "for addr", &w->addr, w->addrlen);
164 #ifndef USE_WINSOCK
165 		close(s);
166 #else
167 		closesocket(s);
168 #endif
169 		return 0;
170 	}
171 #ifdef INET6
172 	if(addr_is_ip6(&w->addr, w->addrlen))
173 		pi = &w->outnet->ip6_ifs[ub_random_max(w->outnet->rnd, num)];
174 	else
175 #endif
176 		pi = &w->outnet->ip4_ifs[ub_random_max(w->outnet->rnd, num)];
177 	log_assert(pi);
178 	if(addr_is_any(&pi->addr, pi->addrlen)) {
179 		/* binding to the ANY interface is for listening sockets */
180 		return 1;
181 	}
182 	/* set port to 0 */
183 	if(addr_is_ip6(&pi->addr, pi->addrlen))
184 		((struct sockaddr_in6*)&pi->addr)->sin6_port = 0;
185 	else	((struct sockaddr_in*)&pi->addr)->sin_port = 0;
186 	if(bind(s, (struct sockaddr*)&pi->addr, pi->addrlen) != 0) {
187 #ifndef USE_WINSOCK
188 		log_err("outgoing tcp: bind: %s", strerror(errno));
189 		close(s);
190 #else
191 		log_err("outgoing tcp: bind: %s",
192 			wsa_strerror(WSAGetLastError()));
193 		closesocket(s);
194 #endif
195 		return 0;
196 	}
197 	log_addr(VERB_ALGO, "tcp bound to src", &pi->addr, pi->addrlen);
198 	return 1;
199 }
200 
201 /** use next free buffer to service a tcp query */
202 static int
203 outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
204 {
205 	struct pending_tcp* pend = w->outnet->tcp_free;
206 	int s;
207 	log_assert(pend);
208 	log_assert(pkt);
209 	log_assert(w->addrlen > 0);
210 	/* open socket */
211 #ifdef INET6
212 	if(addr_is_ip6(&w->addr, w->addrlen))
213 		s = socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP);
214 	else
215 #endif
216 		s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
217 	if(s == -1) {
218 #ifndef USE_WINSOCK
219 		log_err_addr("outgoing tcp: socket", strerror(errno),
220 			&w->addr, w->addrlen);
221 #else
222 		log_err_addr("outgoing tcp: socket",
223 			wsa_strerror(WSAGetLastError()), &w->addr, w->addrlen);
224 #endif
225 		return 0;
226 	}
227 
228 	if (w->outnet->tcp_mss > 0) {
229 #if defined(IPPROTO_TCP) && defined(TCP_MAXSEG)
230 		if(setsockopt(s, IPPROTO_TCP, TCP_MAXSEG,
231 			(void*)&w->outnet->tcp_mss,
232 			(socklen_t)sizeof(w->outnet->tcp_mss)) < 0) {
233 			verbose(VERB_ALGO, "outgoing tcp:"
234 				" setsockopt(.. SO_REUSEADDR ..) failed");
235 		}
236 #else
237 		verbose(VERB_ALGO, "outgoing tcp:"
238 			" setsockopt(TCP_MAXSEG) unsupported");
239 #endif /* defined(IPPROTO_TCP) && defined(TCP_MAXSEG) */
240 	}
241 
242 	if(!pick_outgoing_tcp(w, s))
243 		return 0;
244 
245 	fd_set_nonblock(s);
246 #ifdef USE_OSX_MSG_FASTOPEN
247 	/* API for fast open is different here. We use a connectx() function and
248 	   then writes can happen as normal even using SSL.*/
249 	/* connectx requires that the len be set in the sockaddr struct*/
250 	struct sockaddr_in *addr_in = (struct sockaddr_in *)&w->addr;
251 	addr_in->sin_len = w->addrlen;
252 	sa_endpoints_t endpoints;
253 	endpoints.sae_srcif = 0;
254 	endpoints.sae_srcaddr = NULL;
255 	endpoints.sae_srcaddrlen = 0;
256 	endpoints.sae_dstaddr = (struct sockaddr *)&w->addr;
257 	endpoints.sae_dstaddrlen = w->addrlen;
258 	if (connectx(s, &endpoints, SAE_ASSOCID_ANY,
259 	             CONNECT_DATA_IDEMPOTENT | CONNECT_RESUME_ON_READ_WRITE,
260 	             NULL, 0, NULL, NULL) == -1) {
261 #else /* USE_OSX_MSG_FASTOPEN*/
262 #ifdef USE_MSG_FASTOPEN
263 	pend->c->tcp_do_fastopen = 1;
264 	/* Only do TFO for TCP in which case no connect() is required here.
265 	   Don't combine client TFO with SSL, since OpenSSL can't
266 	   currently support doing a handshake on fd that already isn't connected*/
267 	if (w->outnet->sslctx && w->ssl_upstream) {
268 		if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
269 #else /* USE_MSG_FASTOPEN*/
270 	if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
271 #endif /* USE_MSG_FASTOPEN*/
272 #endif /* USE_OSX_MSG_FASTOPEN*/
273 #ifndef USE_WINSOCK
274 #ifdef EINPROGRESS
275 		if(errno != EINPROGRESS) {
276 #else
277 		if(1) {
278 #endif
279 			if(tcp_connect_errno_needs_log(
280 				(struct sockaddr*)&w->addr, w->addrlen))
281 				log_err_addr("outgoing tcp: connect",
282 					strerror(errno), &w->addr, w->addrlen);
283 			close(s);
284 #else /* USE_WINSOCK */
285 		if(WSAGetLastError() != WSAEINPROGRESS &&
286 			WSAGetLastError() != WSAEWOULDBLOCK) {
287 			closesocket(s);
288 #endif
289 			return 0;
290 		}
291 	}
292 #ifdef USE_MSG_FASTOPEN
293 	}
294 #endif /* USE_MSG_FASTOPEN */
295 	if(w->outnet->sslctx && w->ssl_upstream) {
296 		pend->c->ssl = outgoing_ssl_fd(w->outnet->sslctx, s);
297 		if(!pend->c->ssl) {
298 			pend->c->fd = s;
299 			comm_point_close(pend->c);
300 			return 0;
301 		}
302 #ifdef USE_WINSOCK
303 		comm_point_tcp_win_bio_cb(pend->c, pend->c->ssl);
304 #endif
305 		pend->c->ssl_shake_state = comm_ssl_shake_write;
306 	}
307 	w->pkt = NULL;
308 	w->next_waiting = (void*)pend;
309 	pend->id = LDNS_ID_WIRE(pkt);
310 	w->outnet->num_tcp_outgoing++;
311 	w->outnet->tcp_free = pend->next_free;
312 	pend->next_free = NULL;
313 	pend->query = w;
314 	pend->c->repinfo.addrlen = w->addrlen;
315 	memcpy(&pend->c->repinfo.addr, &w->addr, w->addrlen);
316 	sldns_buffer_clear(pend->c->buffer);
317 	sldns_buffer_write(pend->c->buffer, pkt, pkt_len);
318 	sldns_buffer_flip(pend->c->buffer);
319 	pend->c->tcp_is_reading = 0;
320 	pend->c->tcp_byte_count = 0;
321 	comm_point_start_listening(pend->c, s, -1);
322 	return 1;
323 }
324 
325 /** see if buffers can be used to service TCP queries */
326 static void
327 use_free_buffer(struct outside_network* outnet)
328 {
329 	struct waiting_tcp* w;
330 	while(outnet->tcp_free && outnet->tcp_wait_first
331 		&& !outnet->want_to_quit) {
332 		w = outnet->tcp_wait_first;
333 		outnet->tcp_wait_first = w->next_waiting;
334 		if(outnet->tcp_wait_last == w)
335 			outnet->tcp_wait_last = NULL;
336 		if(!outnet_tcp_take_into_use(w, w->pkt, w->pkt_len)) {
337 			comm_point_callback_t* cb = w->cb;
338 			void* cb_arg = w->cb_arg;
339 			waiting_tcp_delete(w);
340 			fptr_ok(fptr_whitelist_pending_tcp(cb));
341 			(void)(*cb)(NULL, cb_arg, NETEVENT_CLOSED, NULL);
342 		}
343 	}
344 }
345 
346 /** decomission a tcp buffer, closes commpoint and frees waiting_tcp entry */
347 static void
348 decomission_pending_tcp(struct outside_network* outnet,
349 	struct pending_tcp* pend)
350 {
351 	if(pend->c->ssl) {
352 #ifdef HAVE_SSL
353 		SSL_shutdown(pend->c->ssl);
354 		SSL_free(pend->c->ssl);
355 		pend->c->ssl = NULL;
356 #endif
357 	}
358 	comm_point_close(pend->c);
359 	pend->next_free = outnet->tcp_free;
360 	outnet->tcp_free = pend;
361 	waiting_tcp_delete(pend->query);
362 	pend->query = NULL;
363 	use_free_buffer(outnet);
364 }
365 
366 int
367 outnet_tcp_cb(struct comm_point* c, void* arg, int error,
368 	struct comm_reply *reply_info)
369 {
370 	struct pending_tcp* pend = (struct pending_tcp*)arg;
371 	struct outside_network* outnet = pend->query->outnet;
372 	verbose(VERB_ALGO, "outnettcp cb");
373 	if(error != NETEVENT_NOERROR) {
374 		verbose(VERB_QUERY, "outnettcp got tcp error %d", error);
375 		/* pass error below and exit */
376 	} else {
377 		/* check ID */
378 		if(sldns_buffer_limit(c->buffer) < sizeof(uint16_t) ||
379 			LDNS_ID_WIRE(sldns_buffer_begin(c->buffer))!=pend->id) {
380 			log_addr(VERB_QUERY,
381 				"outnettcp: bad ID in reply, from:",
382 				&pend->query->addr, pend->query->addrlen);
383 			error = NETEVENT_CLOSED;
384 		}
385 	}
386 	fptr_ok(fptr_whitelist_pending_tcp(pend->query->cb));
387 	(void)(*pend->query->cb)(c, pend->query->cb_arg, error, reply_info);
388 	decomission_pending_tcp(outnet, pend);
389 	return 0;
390 }
391 
392 /** lower use count on pc, see if it can be closed */
393 static void
394 portcomm_loweruse(struct outside_network* outnet, struct port_comm* pc)
395 {
396 	struct port_if* pif;
397 	pc->num_outstanding--;
398 	if(pc->num_outstanding > 0) {
399 		return;
400 	}
401 	/* close it and replace in unused list */
402 	verbose(VERB_ALGO, "close of port %d", pc->number);
403 	comm_point_close(pc->cp);
404 	pif = pc->pif;
405 	log_assert(pif->inuse > 0);
406 	pif->avail_ports[pif->avail_total - pif->inuse] = pc->number;
407 	pif->inuse--;
408 	pif->out[pc->index] = pif->out[pif->inuse];
409 	pif->out[pc->index]->index = pc->index;
410 	pc->next = outnet->unused_fds;
411 	outnet->unused_fds = pc;
412 }
413 
414 /** try to send waiting UDP queries */
415 static void
416 outnet_send_wait_udp(struct outside_network* outnet)
417 {
418 	struct pending* pend;
419 	/* process waiting queries */
420 	while(outnet->udp_wait_first && outnet->unused_fds
421 		&& !outnet->want_to_quit) {
422 		pend = outnet->udp_wait_first;
423 		outnet->udp_wait_first = pend->next_waiting;
424 		if(!pend->next_waiting) outnet->udp_wait_last = NULL;
425 		sldns_buffer_clear(outnet->udp_buff);
426 		sldns_buffer_write(outnet->udp_buff, pend->pkt, pend->pkt_len);
427 		sldns_buffer_flip(outnet->udp_buff);
428 		free(pend->pkt); /* freeing now makes get_mem correct */
429 		pend->pkt = NULL;
430 		pend->pkt_len = 0;
431 		if(!randomize_and_send_udp(pend, outnet->udp_buff,
432 			pend->timeout)) {
433 			/* callback error on pending */
434 			if(pend->cb) {
435 				fptr_ok(fptr_whitelist_pending_udp(pend->cb));
436 				(void)(*pend->cb)(outnet->unused_fds->cp, pend->cb_arg,
437 					NETEVENT_CLOSED, NULL);
438 			}
439 			pending_delete(outnet, pend);
440 		}
441 	}
442 }
443 
444 int
445 outnet_udp_cb(struct comm_point* c, void* arg, int error,
446 	struct comm_reply *reply_info)
447 {
448 	struct outside_network* outnet = (struct outside_network*)arg;
449 	struct pending key;
450 	struct pending* p;
451 	verbose(VERB_ALGO, "answer cb");
452 
453 	if(error != NETEVENT_NOERROR) {
454 		verbose(VERB_QUERY, "outnetudp got udp error %d", error);
455 		return 0;
456 	}
457 	if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
458 		verbose(VERB_QUERY, "outnetudp udp too short");
459 		return 0;
460 	}
461 	log_assert(reply_info);
462 
463 	/* setup lookup key */
464 	key.id = (unsigned)LDNS_ID_WIRE(sldns_buffer_begin(c->buffer));
465 	memcpy(&key.addr, &reply_info->addr, reply_info->addrlen);
466 	key.addrlen = reply_info->addrlen;
467 	verbose(VERB_ALGO, "Incoming reply id = %4.4x", key.id);
468 	log_addr(VERB_ALGO, "Incoming reply addr =",
469 		&reply_info->addr, reply_info->addrlen);
470 
471 	/* find it, see if this thing is a valid query response */
472 	verbose(VERB_ALGO, "lookup size is %d entries", (int)outnet->pending->count);
473 	p = (struct pending*)rbtree_search(outnet->pending, &key);
474 	if(!p) {
475 		verbose(VERB_QUERY, "received unwanted or unsolicited udp reply dropped.");
476 		log_buf(VERB_ALGO, "dropped message", c->buffer);
477 		outnet->unwanted_replies++;
478 		if(outnet->unwanted_threshold && ++outnet->unwanted_total
479 			>= outnet->unwanted_threshold) {
480 			log_warn("unwanted reply total reached threshold (%u)"
481 				" you may be under attack."
482 				" defensive action: clearing the cache",
483 				(unsigned)outnet->unwanted_threshold);
484 			fptr_ok(fptr_whitelist_alloc_cleanup(
485 				outnet->unwanted_action));
486 			(*outnet->unwanted_action)(outnet->unwanted_param);
487 			outnet->unwanted_total = 0;
488 		}
489 		return 0;
490 	}
491 
492 	verbose(VERB_ALGO, "received udp reply.");
493 	log_buf(VERB_ALGO, "udp message", c->buffer);
494 	if(p->pc->cp != c) {
495 		verbose(VERB_QUERY, "received reply id,addr on wrong port. "
496 			"dropped.");
497 		outnet->unwanted_replies++;
498 		if(outnet->unwanted_threshold && ++outnet->unwanted_total
499 			>= outnet->unwanted_threshold) {
500 			log_warn("unwanted reply total reached threshold (%u)"
501 				" you may be under attack."
502 				" defensive action: clearing the cache",
503 				(unsigned)outnet->unwanted_threshold);
504 			fptr_ok(fptr_whitelist_alloc_cleanup(
505 				outnet->unwanted_action));
506 			(*outnet->unwanted_action)(outnet->unwanted_param);
507 			outnet->unwanted_total = 0;
508 		}
509 		return 0;
510 	}
511 	comm_timer_disable(p->timer);
512 	verbose(VERB_ALGO, "outnet handle udp reply");
513 	/* delete from tree first in case callback creates a retry */
514 	(void)rbtree_delete(outnet->pending, p->node.key);
515 	if(p->cb) {
516 		fptr_ok(fptr_whitelist_pending_udp(p->cb));
517 		(void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_NOERROR, reply_info);
518 	}
519 	portcomm_loweruse(outnet, p->pc);
520 	pending_delete(NULL, p);
521 	outnet_send_wait_udp(outnet);
522 	return 0;
523 }
524 
525 /** calculate number of ip4 and ip6 interfaces*/
526 static void
527 calc_num46(char** ifs, int num_ifs, int do_ip4, int do_ip6,
528 	int* num_ip4, int* num_ip6)
529 {
530 	int i;
531 	*num_ip4 = 0;
532 	*num_ip6 = 0;
533 	if(num_ifs <= 0) {
534 		if(do_ip4)
535 			*num_ip4 = 1;
536 		if(do_ip6)
537 			*num_ip6 = 1;
538 		return;
539 	}
540 	for(i=0; i<num_ifs; i++)
541 	{
542 		if(str_is_ip6(ifs[i])) {
543 			if(do_ip6)
544 				(*num_ip6)++;
545 		} else {
546 			if(do_ip4)
547 				(*num_ip4)++;
548 		}
549 	}
550 
551 }
552 
553 void
554 pending_udp_timer_delay_cb(void* arg)
555 {
556 	struct pending* p = (struct pending*)arg;
557 	struct outside_network* outnet = p->outnet;
558 	verbose(VERB_ALGO, "timeout udp with delay");
559 	portcomm_loweruse(outnet, p->pc);
560 	pending_delete(outnet, p);
561 	outnet_send_wait_udp(outnet);
562 }
563 
564 void
565 pending_udp_timer_cb(void *arg)
566 {
567 	struct pending* p = (struct pending*)arg;
568 	struct outside_network* outnet = p->outnet;
569 	/* it timed out */
570 	verbose(VERB_ALGO, "timeout udp");
571 	if(p->cb) {
572 		fptr_ok(fptr_whitelist_pending_udp(p->cb));
573 		(void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_TIMEOUT, NULL);
574 	}
575 	/* if delayclose, keep port open for a longer time.
576 	 * But if the udpwaitlist exists, then we are struggling to
577 	 * keep up with demand for sockets, so do not wait, but service
578 	 * the customer (customer service more important than portICMPs) */
579 	if(outnet->delayclose && !outnet->udp_wait_first) {
580 		p->cb = NULL;
581 		p->timer->callback = &pending_udp_timer_delay_cb;
582 		comm_timer_set(p->timer, &outnet->delay_tv);
583 		return;
584 	}
585 	portcomm_loweruse(outnet, p->pc);
586 	pending_delete(outnet, p);
587 	outnet_send_wait_udp(outnet);
588 }
589 
590 /** create pending_tcp buffers */
591 static int
592 create_pending_tcp(struct outside_network* outnet, size_t bufsize)
593 {
594 	size_t i;
595 	if(outnet->num_tcp == 0)
596 		return 1; /* no tcp needed, nothing to do */
597 	if(!(outnet->tcp_conns = (struct pending_tcp **)calloc(
598 			outnet->num_tcp, sizeof(struct pending_tcp*))))
599 		return 0;
600 	for(i=0; i<outnet->num_tcp; i++) {
601 		if(!(outnet->tcp_conns[i] = (struct pending_tcp*)calloc(1,
602 			sizeof(struct pending_tcp))))
603 			return 0;
604 		outnet->tcp_conns[i]->next_free = outnet->tcp_free;
605 		outnet->tcp_free = outnet->tcp_conns[i];
606 		outnet->tcp_conns[i]->c = comm_point_create_tcp_out(
607 			outnet->base, bufsize, outnet_tcp_cb,
608 			outnet->tcp_conns[i]);
609 		if(!outnet->tcp_conns[i]->c)
610 			return 0;
611 	}
612 	return 1;
613 }
614 
615 /** setup an outgoing interface, ready address */
616 static int setup_if(struct port_if* pif, const char* addrstr,
617 	int* avail, int numavail, size_t numfd)
618 {
619 	pif->avail_total = numavail;
620 	pif->avail_ports = (int*)memdup(avail, (size_t)numavail*sizeof(int));
621 	if(!pif->avail_ports)
622 		return 0;
623 	if(!ipstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen) &&
624 	   !netblockstrtoaddr(addrstr, UNBOUND_DNS_PORT,
625 			      &pif->addr, &pif->addrlen, &pif->pfxlen))
626 		return 0;
627 	pif->maxout = (int)numfd;
628 	pif->inuse = 0;
629 	pif->out = (struct port_comm**)calloc(numfd,
630 		sizeof(struct port_comm*));
631 	if(!pif->out)
632 		return 0;
633 	return 1;
634 }
635 
636 struct outside_network*
637 outside_network_create(struct comm_base *base, size_t bufsize,
638 	size_t num_ports, char** ifs, int num_ifs, int do_ip4,
639 	int do_ip6, size_t num_tcp, struct infra_cache* infra,
640 	struct ub_randstate* rnd, int use_caps_for_id, int* availports,
641 	int numavailports, size_t unwanted_threshold, int tcp_mss,
642 	void (*unwanted_action)(void*), void* unwanted_param, int do_udp,
643 	void* sslctx, int delayclose, struct dt_env* dtenv)
644 {
645 	struct outside_network* outnet = (struct outside_network*)
646 		calloc(1, sizeof(struct outside_network));
647 	size_t k;
648 	if(!outnet) {
649 		log_err("malloc failed");
650 		return NULL;
651 	}
652 	comm_base_timept(base, &outnet->now_secs, &outnet->now_tv);
653 	outnet->base = base;
654 	outnet->num_tcp = num_tcp;
655 	outnet->num_tcp_outgoing = 0;
656 	outnet->infra = infra;
657 	outnet->rnd = rnd;
658 	outnet->sslctx = sslctx;
659 #ifdef USE_DNSTAP
660 	outnet->dtenv = dtenv;
661 #else
662 	(void)dtenv;
663 #endif
664 	outnet->svcd_overhead = 0;
665 	outnet->want_to_quit = 0;
666 	outnet->unwanted_threshold = unwanted_threshold;
667 	outnet->unwanted_action = unwanted_action;
668 	outnet->unwanted_param = unwanted_param;
669 	outnet->use_caps_for_id = use_caps_for_id;
670 	outnet->do_udp = do_udp;
671 	outnet->tcp_mss = tcp_mss;
672 #ifndef S_SPLINT_S
673 	if(delayclose) {
674 		outnet->delayclose = 1;
675 		outnet->delay_tv.tv_sec = delayclose/1000;
676 		outnet->delay_tv.tv_usec = (delayclose%1000)*1000;
677 	}
678 #endif
679 	if(numavailports == 0) {
680 		log_err("no outgoing ports available");
681 		outside_network_delete(outnet);
682 		return NULL;
683 	}
684 #ifndef INET6
685 	do_ip6 = 0;
686 #endif
687 	calc_num46(ifs, num_ifs, do_ip4, do_ip6,
688 		&outnet->num_ip4, &outnet->num_ip6);
689 	if(outnet->num_ip4 != 0) {
690 		if(!(outnet->ip4_ifs = (struct port_if*)calloc(
691 			(size_t)outnet->num_ip4, sizeof(struct port_if)))) {
692 			log_err("malloc failed");
693 			outside_network_delete(outnet);
694 			return NULL;
695 		}
696 	}
697 	if(outnet->num_ip6 != 0) {
698 		if(!(outnet->ip6_ifs = (struct port_if*)calloc(
699 			(size_t)outnet->num_ip6, sizeof(struct port_if)))) {
700 			log_err("malloc failed");
701 			outside_network_delete(outnet);
702 			return NULL;
703 		}
704 	}
705 	if(	!(outnet->udp_buff = sldns_buffer_new(bufsize)) ||
706 		!(outnet->pending = rbtree_create(pending_cmp)) ||
707 		!(outnet->serviced = rbtree_create(serviced_cmp)) ||
708 		!create_pending_tcp(outnet, bufsize)) {
709 		log_err("malloc failed");
710 		outside_network_delete(outnet);
711 		return NULL;
712 	}
713 
714 	/* allocate commpoints */
715 	for(k=0; k<num_ports; k++) {
716 		struct port_comm* pc;
717 		pc = (struct port_comm*)calloc(1, sizeof(*pc));
718 		if(!pc) {
719 			log_err("malloc failed");
720 			outside_network_delete(outnet);
721 			return NULL;
722 		}
723 		pc->cp = comm_point_create_udp(outnet->base, -1,
724 			outnet->udp_buff, outnet_udp_cb, outnet);
725 		if(!pc->cp) {
726 			log_err("malloc failed");
727 			free(pc);
728 			outside_network_delete(outnet);
729 			return NULL;
730 		}
731 		pc->next = outnet->unused_fds;
732 		outnet->unused_fds = pc;
733 	}
734 
735 	/* allocate interfaces */
736 	if(num_ifs == 0) {
737 		if(do_ip4 && !setup_if(&outnet->ip4_ifs[0], "0.0.0.0",
738 			availports, numavailports, num_ports)) {
739 			log_err("malloc failed");
740 			outside_network_delete(outnet);
741 			return NULL;
742 		}
743 		if(do_ip6 && !setup_if(&outnet->ip6_ifs[0], "::",
744 			availports, numavailports, num_ports)) {
745 			log_err("malloc failed");
746 			outside_network_delete(outnet);
747 			return NULL;
748 		}
749 	} else {
750 		size_t done_4 = 0, done_6 = 0;
751 		int i;
752 		for(i=0; i<num_ifs; i++) {
753 			if(str_is_ip6(ifs[i]) && do_ip6) {
754 				if(!setup_if(&outnet->ip6_ifs[done_6], ifs[i],
755 					availports, numavailports, num_ports)){
756 					log_err("malloc failed");
757 					outside_network_delete(outnet);
758 					return NULL;
759 				}
760 				done_6++;
761 			}
762 			if(!str_is_ip6(ifs[i]) && do_ip4) {
763 				if(!setup_if(&outnet->ip4_ifs[done_4], ifs[i],
764 					availports, numavailports, num_ports)){
765 					log_err("malloc failed");
766 					outside_network_delete(outnet);
767 					return NULL;
768 				}
769 				done_4++;
770 			}
771 		}
772 	}
773 	return outnet;
774 }
775 
776 /** helper pending delete */
777 static void
778 pending_node_del(rbnode_t* node, void* arg)
779 {
780 	struct pending* pend = (struct pending*)node;
781 	struct outside_network* outnet = (struct outside_network*)arg;
782 	pending_delete(outnet, pend);
783 }
784 
785 /** helper serviced delete */
786 static void
787 serviced_node_del(rbnode_t* node, void* ATTR_UNUSED(arg))
788 {
789 	struct serviced_query* sq = (struct serviced_query*)node;
790 	struct service_callback* p = sq->cblist, *np;
791 	free(sq->qbuf);
792 	free(sq->zone);
793 	edns_opt_list_free(sq->opt_list);
794 	while(p) {
795 		np = p->next;
796 		free(p);
797 		p = np;
798 	}
799 	free(sq);
800 }
801 
802 void
803 outside_network_quit_prepare(struct outside_network* outnet)
804 {
805 	if(!outnet)
806 		return;
807 	/* prevent queued items from being sent */
808 	outnet->want_to_quit = 1;
809 }
810 
811 void
812 outside_network_delete(struct outside_network* outnet)
813 {
814 	if(!outnet)
815 		return;
816 	outnet->want_to_quit = 1;
817 	/* check every element, since we can be called on malloc error */
818 	if(outnet->pending) {
819 		/* free pending elements, but do no unlink from tree. */
820 		traverse_postorder(outnet->pending, pending_node_del, NULL);
821 		free(outnet->pending);
822 	}
823 	if(outnet->serviced) {
824 		traverse_postorder(outnet->serviced, serviced_node_del, NULL);
825 		free(outnet->serviced);
826 	}
827 	if(outnet->udp_buff)
828 		sldns_buffer_free(outnet->udp_buff);
829 	if(outnet->unused_fds) {
830 		struct port_comm* p = outnet->unused_fds, *np;
831 		while(p) {
832 			np = p->next;
833 			comm_point_delete(p->cp);
834 			free(p);
835 			p = np;
836 		}
837 		outnet->unused_fds = NULL;
838 	}
839 	if(outnet->ip4_ifs) {
840 		int i, k;
841 		for(i=0; i<outnet->num_ip4; i++) {
842 			for(k=0; k<outnet->ip4_ifs[i].inuse; k++) {
843 				struct port_comm* pc = outnet->ip4_ifs[i].
844 					out[k];
845 				comm_point_delete(pc->cp);
846 				free(pc);
847 			}
848 			free(outnet->ip4_ifs[i].avail_ports);
849 			free(outnet->ip4_ifs[i].out);
850 		}
851 		free(outnet->ip4_ifs);
852 	}
853 	if(outnet->ip6_ifs) {
854 		int i, k;
855 		for(i=0; i<outnet->num_ip6; i++) {
856 			for(k=0; k<outnet->ip6_ifs[i].inuse; k++) {
857 				struct port_comm* pc = outnet->ip6_ifs[i].
858 					out[k];
859 				comm_point_delete(pc->cp);
860 				free(pc);
861 			}
862 			free(outnet->ip6_ifs[i].avail_ports);
863 			free(outnet->ip6_ifs[i].out);
864 		}
865 		free(outnet->ip6_ifs);
866 	}
867 	if(outnet->tcp_conns) {
868 		size_t i;
869 		for(i=0; i<outnet->num_tcp; i++)
870 			if(outnet->tcp_conns[i]) {
871 				comm_point_delete(outnet->tcp_conns[i]->c);
872 				waiting_tcp_delete(outnet->tcp_conns[i]->query);
873 				free(outnet->tcp_conns[i]);
874 			}
875 		free(outnet->tcp_conns);
876 	}
877 	if(outnet->tcp_wait_first) {
878 		struct waiting_tcp* p = outnet->tcp_wait_first, *np;
879 		while(p) {
880 			np = p->next_waiting;
881 			waiting_tcp_delete(p);
882 			p = np;
883 		}
884 	}
885 	if(outnet->udp_wait_first) {
886 		struct pending* p = outnet->udp_wait_first, *np;
887 		while(p) {
888 			np = p->next_waiting;
889 			pending_delete(NULL, p);
890 			p = np;
891 		}
892 	}
893 	free(outnet);
894 }
895 
896 void
897 pending_delete(struct outside_network* outnet, struct pending* p)
898 {
899 	if(!p)
900 		return;
901 	if(outnet && outnet->udp_wait_first &&
902 		(p->next_waiting || p == outnet->udp_wait_last) ) {
903 		/* delete from waiting list, if it is in the waiting list */
904 		struct pending* prev = NULL, *x = outnet->udp_wait_first;
905 		while(x && x != p) {
906 			prev = x;
907 			x = x->next_waiting;
908 		}
909 		if(x) {
910 			log_assert(x == p);
911 			if(prev)
912 				prev->next_waiting = p->next_waiting;
913 			else	outnet->udp_wait_first = p->next_waiting;
914 			if(outnet->udp_wait_last == p)
915 				outnet->udp_wait_last = prev;
916 		}
917 	}
918 	if(outnet) {
919 		(void)rbtree_delete(outnet->pending, p->node.key);
920 	}
921 	if(p->timer)
922 		comm_timer_delete(p->timer);
923 	free(p->pkt);
924 	free(p);
925 }
926 
927 static void
928 sai6_putrandom(struct sockaddr_in6 *sa, int pfxlen, struct ub_randstate *rnd)
929 {
930 	int i, last;
931 	if(!(pfxlen > 0 && pfxlen < 128))
932 		return;
933 	for(i = 0; i < (128 - pfxlen) / 8; i++) {
934 		sa->sin6_addr.s6_addr[15-i] = (uint8_t)ub_random_max(rnd, 256);
935 	}
936 	last = pfxlen & 7;
937 	if(last != 0) {
938 		sa->sin6_addr.s6_addr[15-i] |=
939 			((0xFF >> last) & ub_random_max(rnd, 256));
940 	}
941 }
942 
943 /**
944  * Try to open a UDP socket for outgoing communication.
945  * Sets sockets options as needed.
946  * @param addr: socket address.
947  * @param addrlen: length of address.
948  * @param pfxlen: length of network prefix (for address randomisation).
949  * @param port: port override for addr.
950  * @param inuse: if -1 is returned, this bool means the port was in use.
951  * @param rnd: random state (for address randomisation).
952  * @return fd or -1
953  */
954 static int
955 udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen,
956 	int port, int* inuse, struct ub_randstate* rnd)
957 {
958 	int fd, noproto;
959 	if(addr_is_ip6(addr, addrlen)) {
960 		int freebind = 0;
961 		struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr;
962 		sa.sin6_port = (in_port_t)htons((uint16_t)port);
963 		if(pfxlen != 0) {
964 			freebind = 1;
965 			sai6_putrandom(&sa, pfxlen, rnd);
966 		}
967 		fd = create_udp_sock(AF_INET6, SOCK_DGRAM,
968 			(struct sockaddr*)&sa, addrlen, 1, inuse, &noproto,
969 			0, 0, 0, NULL, 0, freebind);
970 	} else {
971 		struct sockaddr_in* sa = (struct sockaddr_in*)addr;
972 		sa->sin_port = (in_port_t)htons((uint16_t)port);
973 		fd = create_udp_sock(AF_INET, SOCK_DGRAM,
974 			(struct sockaddr*)addr, addrlen, 1, inuse, &noproto,
975 			0, 0, 0, NULL, 0, 0);
976 	}
977 	return fd;
978 }
979 
980 /** Select random ID */
981 static int
982 select_id(struct outside_network* outnet, struct pending* pend,
983 	sldns_buffer* packet)
984 {
985 	int id_tries = 0;
986 	pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
987 	LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
988 
989 	/* insert in tree */
990 	pend->node.key = pend;
991 	while(!rbtree_insert(outnet->pending, &pend->node)) {
992 		/* change ID to avoid collision */
993 		pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
994 		LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
995 		id_tries++;
996 		if(id_tries == MAX_ID_RETRY) {
997 			pend->id=99999; /* non existant ID */
998 			log_err("failed to generate unique ID, drop msg");
999 			return 0;
1000 		}
1001 	}
1002 	verbose(VERB_ALGO, "inserted new pending reply id=%4.4x", pend->id);
1003 	return 1;
1004 }
1005 
1006 /** Select random interface and port */
1007 static int
1008 select_ifport(struct outside_network* outnet, struct pending* pend,
1009 	int num_if, struct port_if* ifs)
1010 {
1011 	int my_if, my_port, fd, portno, inuse, tries=0;
1012 	struct port_if* pif;
1013 	/* randomly select interface and port */
1014 	if(num_if == 0) {
1015 		verbose(VERB_QUERY, "Need to send query but have no "
1016 			"outgoing interfaces of that family");
1017 		return 0;
1018 	}
1019 	log_assert(outnet->unused_fds);
1020 	tries = 0;
1021 	while(1) {
1022 		my_if = ub_random_max(outnet->rnd, num_if);
1023 		pif = &ifs[my_if];
1024 		my_port = ub_random_max(outnet->rnd, pif->avail_total);
1025 		if(my_port < pif->inuse) {
1026 			/* port already open */
1027 			pend->pc = pif->out[my_port];
1028 			verbose(VERB_ALGO, "using UDP if=%d port=%d",
1029 				my_if, pend->pc->number);
1030 			break;
1031 		}
1032 		/* try to open new port, if fails, loop to try again */
1033 		log_assert(pif->inuse < pif->maxout);
1034 		portno = pif->avail_ports[my_port - pif->inuse];
1035 		fd = udp_sockport(&pif->addr, pif->addrlen, pif->pfxlen,
1036 			portno, &inuse, outnet->rnd);
1037 		if(fd == -1 && !inuse) {
1038 			/* nonrecoverable error making socket */
1039 			return 0;
1040 		}
1041 		if(fd != -1) {
1042 			verbose(VERB_ALGO, "opened UDP if=%d port=%d",
1043 				my_if, portno);
1044 			/* grab fd */
1045 			pend->pc = outnet->unused_fds;
1046 			outnet->unused_fds = pend->pc->next;
1047 
1048 			/* setup portcomm */
1049 			pend->pc->next = NULL;
1050 			pend->pc->number = portno;
1051 			pend->pc->pif = pif;
1052 			pend->pc->index = pif->inuse;
1053 			pend->pc->num_outstanding = 0;
1054 			comm_point_start_listening(pend->pc->cp, fd, -1);
1055 
1056 			/* grab port in interface */
1057 			pif->out[pif->inuse] = pend->pc;
1058 			pif->avail_ports[my_port - pif->inuse] =
1059 				pif->avail_ports[pif->avail_total-pif->inuse-1];
1060 			pif->inuse++;
1061 			break;
1062 		}
1063 		/* failed, already in use */
1064 		verbose(VERB_QUERY, "port %d in use, trying another", portno);
1065 		tries++;
1066 		if(tries == MAX_PORT_RETRY) {
1067 			log_err("failed to find an open port, drop msg");
1068 			return 0;
1069 		}
1070 	}
1071 	log_assert(pend->pc);
1072 	pend->pc->num_outstanding++;
1073 
1074 	return 1;
1075 }
1076 
1077 static int
1078 randomize_and_send_udp(struct pending* pend, sldns_buffer* packet, int timeout)
1079 {
1080 	struct timeval tv;
1081 	struct outside_network* outnet = pend->sq->outnet;
1082 
1083 	/* select id */
1084 	if(!select_id(outnet, pend, packet)) {
1085 		return 0;
1086 	}
1087 
1088 	/* select src_if, port */
1089 	if(addr_is_ip6(&pend->addr, pend->addrlen)) {
1090 		if(!select_ifport(outnet, pend,
1091 			outnet->num_ip6, outnet->ip6_ifs))
1092 			return 0;
1093 	} else {
1094 		if(!select_ifport(outnet, pend,
1095 			outnet->num_ip4, outnet->ip4_ifs))
1096 			return 0;
1097 	}
1098 	log_assert(pend->pc && pend->pc->cp);
1099 
1100 	/* send it over the commlink */
1101 	if(!comm_point_send_udp_msg(pend->pc->cp, packet,
1102 		(struct sockaddr*)&pend->addr, pend->addrlen)) {
1103 		portcomm_loweruse(outnet, pend->pc);
1104 		return 0;
1105 	}
1106 
1107 	/* system calls to set timeout after sending UDP to make roundtrip
1108 	   smaller. */
1109 #ifndef S_SPLINT_S
1110 	tv.tv_sec = timeout/1000;
1111 	tv.tv_usec = (timeout%1000)*1000;
1112 #endif
1113 	comm_timer_set(pend->timer, &tv);
1114 
1115 #ifdef USE_DNSTAP
1116 	if(outnet->dtenv &&
1117 	   (outnet->dtenv->log_resolver_query_messages ||
1118 	    outnet->dtenv->log_forwarder_query_messages))
1119 		dt_msg_send_outside_query(outnet->dtenv, &pend->addr, comm_udp,
1120 		pend->sq->zone, pend->sq->zonelen, packet);
1121 #endif
1122 	return 1;
1123 }
1124 
1125 struct pending*
1126 pending_udp_query(struct serviced_query* sq, struct sldns_buffer* packet,
1127 	int timeout, comm_point_callback_t* cb, void* cb_arg)
1128 {
1129 	struct pending* pend = (struct pending*)calloc(1, sizeof(*pend));
1130 	if(!pend) return NULL;
1131 	pend->outnet = sq->outnet;
1132 	pend->sq = sq;
1133 	pend->addrlen = sq->addrlen;
1134 	memmove(&pend->addr, &sq->addr, sq->addrlen);
1135 	pend->cb = cb;
1136 	pend->cb_arg = cb_arg;
1137 	pend->node.key = pend;
1138 	pend->timer = comm_timer_create(sq->outnet->base, pending_udp_timer_cb,
1139 		pend);
1140 	if(!pend->timer) {
1141 		free(pend);
1142 		return NULL;
1143 	}
1144 
1145 	if(sq->outnet->unused_fds == NULL) {
1146 		/* no unused fd, cannot create a new port (randomly) */
1147 		verbose(VERB_ALGO, "no fds available, udp query waiting");
1148 		pend->timeout = timeout;
1149 		pend->pkt_len = sldns_buffer_limit(packet);
1150 		pend->pkt = (uint8_t*)memdup(sldns_buffer_begin(packet),
1151 			pend->pkt_len);
1152 		if(!pend->pkt) {
1153 			comm_timer_delete(pend->timer);
1154 			free(pend);
1155 			return NULL;
1156 		}
1157 		/* put at end of waiting list */
1158 		if(sq->outnet->udp_wait_last)
1159 			sq->outnet->udp_wait_last->next_waiting = pend;
1160 		else
1161 			sq->outnet->udp_wait_first = pend;
1162 		sq->outnet->udp_wait_last = pend;
1163 		return pend;
1164 	}
1165 	if(!randomize_and_send_udp(pend, packet, timeout)) {
1166 		pending_delete(sq->outnet, pend);
1167 		return NULL;
1168 	}
1169 	return pend;
1170 }
1171 
1172 void
1173 outnet_tcptimer(void* arg)
1174 {
1175 	struct waiting_tcp* w = (struct waiting_tcp*)arg;
1176 	struct outside_network* outnet = w->outnet;
1177 	comm_point_callback_t* cb;
1178 	void* cb_arg;
1179 	if(w->pkt) {
1180 		/* it is on the waiting list */
1181 		waiting_list_remove(outnet, w);
1182 	} else {
1183 		/* it was in use */
1184 		struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
1185 		comm_point_close(pend->c);
1186 		pend->query = NULL;
1187 		pend->next_free = outnet->tcp_free;
1188 		outnet->tcp_free = pend;
1189 	}
1190 	cb = w->cb;
1191 	cb_arg = w->cb_arg;
1192 	waiting_tcp_delete(w);
1193 	fptr_ok(fptr_whitelist_pending_tcp(cb));
1194 	(void)(*cb)(NULL, cb_arg, NETEVENT_TIMEOUT, NULL);
1195 	use_free_buffer(outnet);
1196 }
1197 
1198 struct waiting_tcp*
1199 pending_tcp_query(struct serviced_query* sq, sldns_buffer* packet,
1200 	int timeout, comm_point_callback_t* callback, void* callback_arg)
1201 {
1202 	struct pending_tcp* pend = sq->outnet->tcp_free;
1203 	struct waiting_tcp* w;
1204 	struct timeval tv;
1205 	uint16_t id;
1206 	/* if no buffer is free allocate space to store query */
1207 	w = (struct waiting_tcp*)malloc(sizeof(struct waiting_tcp)
1208 		+ (pend?0:sldns_buffer_limit(packet)));
1209 	if(!w) {
1210 		return NULL;
1211 	}
1212 	if(!(w->timer = comm_timer_create(sq->outnet->base, outnet_tcptimer, w))) {
1213 		free(w);
1214 		return NULL;
1215 	}
1216 	w->pkt = NULL;
1217 	w->pkt_len = 0;
1218 	id = ((unsigned)ub_random(sq->outnet->rnd)>>8) & 0xffff;
1219 	LDNS_ID_SET(sldns_buffer_begin(packet), id);
1220 	memcpy(&w->addr, &sq->addr, sq->addrlen);
1221 	w->addrlen = sq->addrlen;
1222 	w->outnet = sq->outnet;
1223 	w->cb = callback;
1224 	w->cb_arg = callback_arg;
1225 	w->ssl_upstream = sq->ssl_upstream;
1226 #ifndef S_SPLINT_S
1227 	tv.tv_sec = timeout;
1228 	tv.tv_usec = 0;
1229 #endif
1230 	comm_timer_set(w->timer, &tv);
1231 	if(pend) {
1232 		/* we have a buffer available right now */
1233 		if(!outnet_tcp_take_into_use(w, sldns_buffer_begin(packet),
1234 			sldns_buffer_limit(packet))) {
1235 			waiting_tcp_delete(w);
1236 			return NULL;
1237 		}
1238 #ifdef USE_DNSTAP
1239 		if(sq->outnet->dtenv &&
1240 		   (sq->outnet->dtenv->log_resolver_query_messages ||
1241 		    sq->outnet->dtenv->log_forwarder_query_messages))
1242 		dt_msg_send_outside_query(sq->outnet->dtenv, &sq->addr,
1243 		comm_tcp, sq->zone, sq->zonelen, packet);
1244 #endif
1245 	} else {
1246 		/* queue up */
1247 		w->pkt = (uint8_t*)w + sizeof(struct waiting_tcp);
1248 		w->pkt_len = sldns_buffer_limit(packet);
1249 		memmove(w->pkt, sldns_buffer_begin(packet), w->pkt_len);
1250 		w->next_waiting = NULL;
1251 		if(sq->outnet->tcp_wait_last)
1252 			sq->outnet->tcp_wait_last->next_waiting = w;
1253 		else	sq->outnet->tcp_wait_first = w;
1254 		sq->outnet->tcp_wait_last = w;
1255 	}
1256 	return w;
1257 }
1258 
1259 /** create query for serviced queries */
1260 static void
1261 serviced_gen_query(sldns_buffer* buff, uint8_t* qname, size_t qnamelen,
1262 	uint16_t qtype, uint16_t qclass, uint16_t flags)
1263 {
1264 	sldns_buffer_clear(buff);
1265 	/* skip id */
1266 	sldns_buffer_write_u16(buff, flags);
1267 	sldns_buffer_write_u16(buff, 1); /* qdcount */
1268 	sldns_buffer_write_u16(buff, 0); /* ancount */
1269 	sldns_buffer_write_u16(buff, 0); /* nscount */
1270 	sldns_buffer_write_u16(buff, 0); /* arcount */
1271 	sldns_buffer_write(buff, qname, qnamelen);
1272 	sldns_buffer_write_u16(buff, qtype);
1273 	sldns_buffer_write_u16(buff, qclass);
1274 	sldns_buffer_flip(buff);
1275 }
1276 
1277 /** lookup serviced query in serviced query rbtree */
1278 static struct serviced_query*
1279 lookup_serviced(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1280 	struct sockaddr_storage* addr, socklen_t addrlen,
1281 	struct edns_option* opt_list)
1282 {
1283 	struct serviced_query key;
1284 	key.node.key = &key;
1285 	key.qbuf = sldns_buffer_begin(buff);
1286 	key.qbuflen = sldns_buffer_limit(buff);
1287 	key.dnssec = dnssec;
1288 	memcpy(&key.addr, addr, addrlen);
1289 	key.addrlen = addrlen;
1290 	key.outnet = outnet;
1291 	key.opt_list = opt_list;
1292 	return (struct serviced_query*)rbtree_search(outnet->serviced, &key);
1293 }
1294 
1295 /** Create new serviced entry */
1296 static struct serviced_query*
1297 serviced_create(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1298 	int want_dnssec, int nocaps, int tcp_upstream, int ssl_upstream,
1299 	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
1300 	size_t zonelen, int qtype, struct edns_option* opt_list)
1301 {
1302 	struct serviced_query* sq = (struct serviced_query*)malloc(sizeof(*sq));
1303 #ifdef UNBOUND_DEBUG
1304 	rbnode_t* ins;
1305 #endif
1306 	if(!sq)
1307 		return NULL;
1308 	sq->node.key = sq;
1309 	sq->qbuf = memdup(sldns_buffer_begin(buff), sldns_buffer_limit(buff));
1310 	if(!sq->qbuf) {
1311 		free(sq);
1312 		return NULL;
1313 	}
1314 	sq->qbuflen = sldns_buffer_limit(buff);
1315 	sq->zone = memdup(zone, zonelen);
1316 	if(!sq->zone) {
1317 		free(sq->qbuf);
1318 		free(sq);
1319 		return NULL;
1320 	}
1321 	sq->zonelen = zonelen;
1322 	sq->qtype = qtype;
1323 	sq->dnssec = dnssec;
1324 	sq->want_dnssec = want_dnssec;
1325 	sq->nocaps = nocaps;
1326 	sq->tcp_upstream = tcp_upstream;
1327 	sq->ssl_upstream = ssl_upstream;
1328 	memcpy(&sq->addr, addr, addrlen);
1329 	sq->addrlen = addrlen;
1330 	sq->opt_list = NULL;
1331 	if(opt_list) {
1332 		sq->opt_list = edns_opt_copy_alloc(opt_list);
1333 		if(!sq->opt_list) {
1334 			free(sq->zone);
1335 			free(sq->qbuf);
1336 			free(sq);
1337 			return NULL;
1338 		}
1339 	}
1340 	sq->outnet = outnet;
1341 	sq->cblist = NULL;
1342 	sq->pending = NULL;
1343 	sq->status = serviced_initial;
1344 	sq->retry = 0;
1345 	sq->to_be_deleted = 0;
1346 #ifdef UNBOUND_DEBUG
1347 	ins =
1348 #else
1349 	(void)
1350 #endif
1351 	rbtree_insert(outnet->serviced, &sq->node);
1352 	log_assert(ins != NULL); /* must not be already present */
1353 	return sq;
1354 }
1355 
1356 /** remove waiting tcp from the outnet waiting list */
1357 static void
1358 waiting_list_remove(struct outside_network* outnet, struct waiting_tcp* w)
1359 {
1360 	struct waiting_tcp* p = outnet->tcp_wait_first, *prev = NULL;
1361 	while(p) {
1362 		if(p == w) {
1363 			/* remove w */
1364 			if(prev)
1365 				prev->next_waiting = w->next_waiting;
1366 			else	outnet->tcp_wait_first = w->next_waiting;
1367 			if(outnet->tcp_wait_last == w)
1368 				outnet->tcp_wait_last = prev;
1369 			return;
1370 		}
1371 		prev = p;
1372 		p = p->next_waiting;
1373 	}
1374 }
1375 
1376 /** cleanup serviced query entry */
1377 static void
1378 serviced_delete(struct serviced_query* sq)
1379 {
1380 	if(sq->pending) {
1381 		/* clear up the pending query */
1382 		if(sq->status == serviced_query_UDP_EDNS ||
1383 			sq->status == serviced_query_UDP ||
1384 			sq->status == serviced_query_PROBE_EDNS ||
1385 			sq->status == serviced_query_UDP_EDNS_FRAG ||
1386 			sq->status == serviced_query_UDP_EDNS_fallback) {
1387 			struct pending* p = (struct pending*)sq->pending;
1388 			if(p->pc)
1389 				portcomm_loweruse(sq->outnet, p->pc);
1390 			pending_delete(sq->outnet, p);
1391 			/* this call can cause reentrant calls back into the
1392 			 * mesh */
1393 			outnet_send_wait_udp(sq->outnet);
1394 		} else {
1395 			struct waiting_tcp* p = (struct waiting_tcp*)
1396 				sq->pending;
1397 			if(p->pkt == NULL) {
1398 				decomission_pending_tcp(sq->outnet,
1399 					(struct pending_tcp*)p->next_waiting);
1400 			} else {
1401 				waiting_list_remove(sq->outnet, p);
1402 				waiting_tcp_delete(p);
1403 			}
1404 		}
1405 	}
1406 	/* does not delete from tree, caller has to do that */
1407 	serviced_node_del(&sq->node, NULL);
1408 }
1409 
1410 /** perturb a dname capitalization randomly */
1411 static void
1412 serviced_perturb_qname(struct ub_randstate* rnd, uint8_t* qbuf, size_t len)
1413 {
1414 	uint8_t lablen;
1415 	uint8_t* d = qbuf + 10;
1416 	long int random = 0;
1417 	int bits = 0;
1418 	log_assert(len >= 10 + 5 /* offset qname, root, qtype, qclass */);
1419 	(void)len;
1420 	lablen = *d++;
1421 	while(lablen) {
1422 		while(lablen--) {
1423 			/* only perturb A-Z, a-z */
1424 			if(isalpha((unsigned char)*d)) {
1425 				/* get a random bit */
1426 				if(bits == 0) {
1427 					random = ub_random(rnd);
1428 					bits = 30;
1429 				}
1430 				if(random & 0x1) {
1431 					*d = (uint8_t)toupper((unsigned char)*d);
1432 				} else {
1433 					*d = (uint8_t)tolower((unsigned char)*d);
1434 				}
1435 				random >>= 1;
1436 				bits--;
1437 			}
1438 			d++;
1439 		}
1440 		lablen = *d++;
1441 	}
1442 	if(verbosity >= VERB_ALGO) {
1443 		char buf[LDNS_MAX_DOMAINLEN+1];
1444 		dname_str(qbuf+10, buf);
1445 		verbose(VERB_ALGO, "qname perturbed to %s", buf);
1446 	}
1447 }
1448 
1449 /** put serviced query into a buffer */
1450 static void
1451 serviced_encode(struct serviced_query* sq, sldns_buffer* buff, int with_edns)
1452 {
1453 	/* if we are using 0x20 bits for ID randomness, perturb them */
1454 	if(sq->outnet->use_caps_for_id && !sq->nocaps) {
1455 		serviced_perturb_qname(sq->outnet->rnd, sq->qbuf, sq->qbuflen);
1456 	}
1457 	/* generate query */
1458 	sldns_buffer_clear(buff);
1459 	sldns_buffer_write_u16(buff, 0); /* id placeholder */
1460 	sldns_buffer_write(buff, sq->qbuf, sq->qbuflen);
1461 	sldns_buffer_flip(buff);
1462 	if(with_edns) {
1463 		/* add edns section */
1464 		struct edns_data edns;
1465 		edns.edns_present = 1;
1466 		edns.ext_rcode = 0;
1467 		edns.edns_version = EDNS_ADVERTISED_VERSION;
1468 		edns.opt_list = sq->opt_list;
1469 		if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1470 			if(addr_is_ip6(&sq->addr, sq->addrlen)) {
1471 				if(EDNS_FRAG_SIZE_IP6 < EDNS_ADVERTISED_SIZE)
1472 					edns.udp_size = EDNS_FRAG_SIZE_IP6;
1473 				else	edns.udp_size = EDNS_ADVERTISED_SIZE;
1474 			} else {
1475 				if(EDNS_FRAG_SIZE_IP4 < EDNS_ADVERTISED_SIZE)
1476 					edns.udp_size = EDNS_FRAG_SIZE_IP4;
1477 				else	edns.udp_size = EDNS_ADVERTISED_SIZE;
1478 			}
1479 		} else {
1480 			edns.udp_size = EDNS_ADVERTISED_SIZE;
1481 		}
1482 		edns.bits = 0;
1483 		if(sq->dnssec & EDNS_DO)
1484 			edns.bits = EDNS_DO;
1485 		if(sq->dnssec & BIT_CD)
1486 			LDNS_CD_SET(sldns_buffer_begin(buff));
1487 		attach_edns_record(buff, &edns);
1488 	}
1489 }
1490 
1491 /**
1492  * Perform serviced query UDP sending operation.
1493  * Sends UDP with EDNS, unless infra host marked non EDNS.
1494  * @param sq: query to send.
1495  * @param buff: buffer scratch space.
1496  * @return 0 on error.
1497  */
1498 static int
1499 serviced_udp_send(struct serviced_query* sq, sldns_buffer* buff)
1500 {
1501 	int rtt, vs;
1502 	uint8_t edns_lame_known;
1503 	time_t now = *sq->outnet->now_secs;
1504 
1505 	if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1506 		sq->zonelen, now, &vs, &edns_lame_known, &rtt))
1507 		return 0;
1508 	sq->last_rtt = rtt;
1509 	verbose(VERB_ALGO, "EDNS lookup known=%d vs=%d", edns_lame_known, vs);
1510 	if(sq->status == serviced_initial) {
1511 		if(edns_lame_known == 0 && rtt > 5000 && rtt < 10001) {
1512 			/* perform EDNS lame probe - check if server is
1513 			 * EDNS lame (EDNS queries to it are dropped) */
1514 			verbose(VERB_ALGO, "serviced query: send probe to see "
1515 				" if use of EDNS causes timeouts");
1516 			/* even 700 msec may be too small */
1517 			rtt = 1000;
1518 			sq->status = serviced_query_PROBE_EDNS;
1519 		} else if(vs != -1) {
1520 			sq->status = serviced_query_UDP_EDNS;
1521 		} else {
1522 			sq->status = serviced_query_UDP;
1523 		}
1524 	}
1525 	serviced_encode(sq, buff, (sq->status == serviced_query_UDP_EDNS) ||
1526 		(sq->status == serviced_query_UDP_EDNS_FRAG));
1527 	sq->last_sent_time = *sq->outnet->now_tv;
1528 	sq->edns_lame_known = (int)edns_lame_known;
1529 	verbose(VERB_ALGO, "serviced query UDP timeout=%d msec", rtt);
1530 	sq->pending = pending_udp_query(sq, buff, rtt,
1531 		serviced_udp_callback, sq);
1532 	if(!sq->pending)
1533 		return 0;
1534 	return 1;
1535 }
1536 
1537 /** check that perturbed qname is identical */
1538 static int
1539 serviced_check_qname(sldns_buffer* pkt, uint8_t* qbuf, size_t qbuflen)
1540 {
1541 	uint8_t* d1 = sldns_buffer_at(pkt, 12);
1542 	uint8_t* d2 = qbuf+10;
1543 	uint8_t len1, len2;
1544 	int count = 0;
1545 	log_assert(qbuflen >= 15 /* 10 header, root, type, class */);
1546 	len1 = *d1++;
1547 	len2 = *d2++;
1548 	if(sldns_buffer_limit(pkt) < 12+1+4) /* packet too small for qname */
1549 		return 0;
1550 	while(len1 != 0 || len2 != 0) {
1551 		if(LABEL_IS_PTR(len1)) {
1552 			d1 = sldns_buffer_at(pkt, PTR_OFFSET(len1, *d1));
1553 			if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1554 				return 0;
1555 			len1 = *d1++;
1556 			if(count++ > MAX_COMPRESS_PTRS)
1557 				return 0;
1558 			continue;
1559 		}
1560 		if(d2 > qbuf+qbuflen)
1561 			return 0;
1562 		if(len1 != len2)
1563 			return 0;
1564 		if(len1 > LDNS_MAX_LABELLEN)
1565 			return 0;
1566 		log_assert(len1 <= LDNS_MAX_LABELLEN);
1567 		log_assert(len2 <= LDNS_MAX_LABELLEN);
1568 		log_assert(len1 == len2 && len1 != 0);
1569 		/* compare the labels - bitwise identical */
1570 		if(memcmp(d1, d2, len1) != 0)
1571 			return 0;
1572 		d1 += len1;
1573 		d2 += len2;
1574 		len1 = *d1++;
1575 		len2 = *d2++;
1576 	}
1577 	return 1;
1578 }
1579 
1580 /** call the callbacks for a serviced query */
1581 static void
1582 serviced_callbacks(struct serviced_query* sq, int error, struct comm_point* c,
1583 	struct comm_reply* rep)
1584 {
1585 	struct service_callback* p;
1586 	int dobackup = (sq->cblist && sq->cblist->next); /* >1 cb*/
1587 	uint8_t *backup_p = NULL;
1588 	size_t backlen = 0;
1589 #ifdef UNBOUND_DEBUG
1590 	rbnode_t* rem =
1591 #else
1592 	(void)
1593 #endif
1594 	/* remove from tree, and schedule for deletion, so that callbacks
1595 	 * can safely deregister themselves and even create new serviced
1596 	 * queries that are identical to this one. */
1597 	rbtree_delete(sq->outnet->serviced, sq);
1598 	log_assert(rem); /* should have been present */
1599 	sq->to_be_deleted = 1;
1600 	verbose(VERB_ALGO, "svcd callbacks start");
1601 	if(sq->outnet->use_caps_for_id && error == NETEVENT_NOERROR && c &&
1602 		!sq->nocaps && sq->qtype != LDNS_RR_TYPE_PTR) {
1603 		/* for type PTR do not check perturbed name in answer,
1604 		 * compatibility with cisco dns guard boxes that mess up
1605 		 * reverse queries 0x20 contents */
1606 		/* noerror and nxdomain must have a qname in reply */
1607 		if(sldns_buffer_read_u16_at(c->buffer, 4) == 0 &&
1608 			(LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1609 				== LDNS_RCODE_NOERROR ||
1610 			 LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1611 				== LDNS_RCODE_NXDOMAIN)) {
1612 			verbose(VERB_DETAIL, "no qname in reply to check 0x20ID");
1613 			log_addr(VERB_DETAIL, "from server",
1614 				&sq->addr, sq->addrlen);
1615 			log_buf(VERB_DETAIL, "for packet", c->buffer);
1616 			error = NETEVENT_CLOSED;
1617 			c = NULL;
1618 		} else if(sldns_buffer_read_u16_at(c->buffer, 4) > 0 &&
1619 			!serviced_check_qname(c->buffer, sq->qbuf,
1620 			sq->qbuflen)) {
1621 			verbose(VERB_DETAIL, "wrong 0x20-ID in reply qname");
1622 			log_addr(VERB_DETAIL, "from server",
1623 				&sq->addr, sq->addrlen);
1624 			log_buf(VERB_DETAIL, "for packet", c->buffer);
1625 			error = NETEVENT_CAPSFAIL;
1626 			/* and cleanup too */
1627 			pkt_dname_tolower(c->buffer,
1628 				sldns_buffer_at(c->buffer, 12));
1629 		} else {
1630 			verbose(VERB_ALGO, "good 0x20-ID in reply qname");
1631 			/* cleanup caps, prettier cache contents. */
1632 			pkt_dname_tolower(c->buffer,
1633 				sldns_buffer_at(c->buffer, 12));
1634 		}
1635 	}
1636 	if(dobackup && c) {
1637 		/* make a backup of the query, since the querystate processing
1638 		 * may send outgoing queries that overwrite the buffer.
1639 		 * use secondary buffer to store the query.
1640 		 * This is a data copy, but faster than packet to server */
1641 		backlen = sldns_buffer_limit(c->buffer);
1642 		backup_p = memdup(sldns_buffer_begin(c->buffer), backlen);
1643 		if(!backup_p) {
1644 			log_err("malloc failure in serviced query callbacks");
1645 			error = NETEVENT_CLOSED;
1646 			c = NULL;
1647 		}
1648 		sq->outnet->svcd_overhead = backlen;
1649 	}
1650 	/* test the actual sq->cblist, because the next elem could be deleted*/
1651 	while((p=sq->cblist) != NULL) {
1652 		sq->cblist = p->next; /* remove this element */
1653 		if(dobackup && c) {
1654 			sldns_buffer_clear(c->buffer);
1655 			sldns_buffer_write(c->buffer, backup_p, backlen);
1656 			sldns_buffer_flip(c->buffer);
1657 		}
1658 		fptr_ok(fptr_whitelist_serviced_query(p->cb));
1659 		(void)(*p->cb)(c, p->cb_arg, error, rep);
1660 		free(p);
1661 	}
1662 	if(backup_p) {
1663 		free(backup_p);
1664 		sq->outnet->svcd_overhead = 0;
1665 	}
1666 	verbose(VERB_ALGO, "svcd callbacks end");
1667 	log_assert(sq->cblist == NULL);
1668 	serviced_delete(sq);
1669 }
1670 
1671 int
1672 serviced_tcp_callback(struct comm_point* c, void* arg, int error,
1673         struct comm_reply* rep)
1674 {
1675 	struct serviced_query* sq = (struct serviced_query*)arg;
1676 	struct comm_reply r2;
1677 	sq->pending = NULL; /* removed after this callback */
1678 	if(error != NETEVENT_NOERROR)
1679 		log_addr(VERB_QUERY, "tcp error for address",
1680 			&sq->addr, sq->addrlen);
1681 	if(error==NETEVENT_NOERROR)
1682 		infra_update_tcp_works(sq->outnet->infra, &sq->addr,
1683 			sq->addrlen, sq->zone, sq->zonelen);
1684 #ifdef USE_DNSTAP
1685 	if(error==NETEVENT_NOERROR && sq->outnet->dtenv &&
1686 	   (sq->outnet->dtenv->log_resolver_response_messages ||
1687 	    sq->outnet->dtenv->log_forwarder_response_messages))
1688 		dt_msg_send_outside_response(sq->outnet->dtenv, &sq->addr,
1689 		c->type, sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
1690 		&sq->last_sent_time, sq->outnet->now_tv, c->buffer);
1691 #endif
1692 	if(error==NETEVENT_NOERROR && sq->status == serviced_query_TCP_EDNS &&
1693 		(LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1694 		LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(sldns_buffer_begin(
1695 		c->buffer)) == LDNS_RCODE_NOTIMPL) ) {
1696 		/* attempt to fallback to nonEDNS */
1697 		sq->status = serviced_query_TCP_EDNS_fallback;
1698 		serviced_tcp_initiate(sq, c->buffer);
1699 		return 0;
1700 	} else if(error==NETEVENT_NOERROR &&
1701 		sq->status == serviced_query_TCP_EDNS_fallback &&
1702 			(LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1703 			LDNS_RCODE_NOERROR || LDNS_RCODE_WIRE(
1704 			sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NXDOMAIN
1705 			|| LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1706 			== LDNS_RCODE_YXDOMAIN)) {
1707 		/* the fallback produced a result that looks promising, note
1708 		 * that this server should be approached without EDNS */
1709 		/* only store noEDNS in cache if domain is noDNSSEC */
1710 		if(!sq->want_dnssec)
1711 		  if(!infra_edns_update(sq->outnet->infra, &sq->addr,
1712 			sq->addrlen, sq->zone, sq->zonelen, -1,
1713 			*sq->outnet->now_secs))
1714 			log_err("Out of memory caching no edns for host");
1715 		sq->status = serviced_query_TCP;
1716 	}
1717 	if(sq->tcp_upstream || sq->ssl_upstream) {
1718 	    struct timeval now = *sq->outnet->now_tv;
1719 	    if(now.tv_sec > sq->last_sent_time.tv_sec ||
1720 		(now.tv_sec == sq->last_sent_time.tv_sec &&
1721 		now.tv_usec > sq->last_sent_time.tv_usec)) {
1722 		/* convert from microseconds to milliseconds */
1723 		int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
1724 		  + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
1725 		verbose(VERB_ALGO, "measured TCP-time at %d msec", roundtime);
1726 		log_assert(roundtime >= 0);
1727 		/* only store if less then AUTH_TIMEOUT seconds, it could be
1728 		 * huge due to system-hibernated and we woke up */
1729 		if(roundtime < TCP_AUTH_QUERY_TIMEOUT*1000) {
1730 		    if(!infra_rtt_update(sq->outnet->infra, &sq->addr,
1731 			sq->addrlen, sq->zone, sq->zonelen, sq->qtype,
1732 			roundtime, sq->last_rtt, (time_t)now.tv_sec))
1733 			log_err("out of memory noting rtt.");
1734 		}
1735 	    }
1736 	}
1737 	/* insert address into reply info */
1738 	if(!rep) {
1739 		/* create one if there isn't (on errors) */
1740 		rep = &r2;
1741 		r2.c = c;
1742 	}
1743 	memcpy(&rep->addr, &sq->addr, sq->addrlen);
1744 	rep->addrlen = sq->addrlen;
1745 	serviced_callbacks(sq, error, c, rep);
1746 	return 0;
1747 }
1748 
1749 static void
1750 serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff)
1751 {
1752 	verbose(VERB_ALGO, "initiate TCP query %s",
1753 		sq->status==serviced_query_TCP_EDNS?"EDNS":"");
1754 	serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1755 	sq->last_sent_time = *sq->outnet->now_tv;
1756 	sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
1757 		serviced_tcp_callback, sq);
1758 	if(!sq->pending) {
1759 		/* delete from tree so that a retry by above layer does not
1760 		 * clash with this entry */
1761 		log_err("serviced_tcp_initiate: failed to send tcp query");
1762 		serviced_callbacks(sq, NETEVENT_CLOSED, NULL, NULL);
1763 	}
1764 }
1765 
1766 /** Send serviced query over TCP return false on initial failure */
1767 static int
1768 serviced_tcp_send(struct serviced_query* sq, sldns_buffer* buff)
1769 {
1770 	int vs, rtt;
1771 	uint8_t edns_lame_known;
1772 	if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1773 		sq->zonelen, *sq->outnet->now_secs, &vs, &edns_lame_known,
1774 		&rtt))
1775 		return 0;
1776 	if(vs != -1)
1777 		sq->status = serviced_query_TCP_EDNS;
1778 	else 	sq->status = serviced_query_TCP;
1779 	serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1780 	sq->last_sent_time = *sq->outnet->now_tv;
1781 	sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
1782 		serviced_tcp_callback, sq);
1783 	return sq->pending != NULL;
1784 }
1785 
1786 /* see if packet is edns malformed; got zeroes at start.
1787  * This is from servers that return malformed packets to EDNS0 queries,
1788  * but they return good packets for nonEDNS0 queries.
1789  * We try to detect their output; without resorting to a full parse or
1790  * check for too many bytes after the end of the packet. */
1791 static int
1792 packet_edns_malformed(struct sldns_buffer* buf, int qtype)
1793 {
1794 	size_t len;
1795 	if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE)
1796 		return 1; /* malformed */
1797 	/* they have NOERROR rcode, 1 answer. */
1798 	if(LDNS_RCODE_WIRE(sldns_buffer_begin(buf)) != LDNS_RCODE_NOERROR)
1799 		return 0;
1800 	/* one query (to skip) and answer records */
1801 	if(LDNS_QDCOUNT(sldns_buffer_begin(buf)) != 1 ||
1802 		LDNS_ANCOUNT(sldns_buffer_begin(buf)) == 0)
1803 		return 0;
1804 	/* skip qname */
1805 	len = dname_valid(sldns_buffer_at(buf, LDNS_HEADER_SIZE),
1806 		sldns_buffer_limit(buf)-LDNS_HEADER_SIZE);
1807 	if(len == 0)
1808 		return 0;
1809 	if(len == 1 && qtype == 0)
1810 		return 0; /* we asked for '.' and type 0 */
1811 	/* and then 4 bytes (type and class of query) */
1812 	if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE + len + 4 + 3)
1813 		return 0;
1814 
1815 	/* and start with 11 zeroes as the answer RR */
1816 	/* so check the qtype of the answer record, qname=0, type=0 */
1817 	if(sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[0] == 0 &&
1818 	   sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[1] == 0 &&
1819 	   sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[2] == 0)
1820 		return 1;
1821 	return 0;
1822 }
1823 
1824 int
1825 serviced_udp_callback(struct comm_point* c, void* arg, int error,
1826         struct comm_reply* rep)
1827 {
1828 	struct serviced_query* sq = (struct serviced_query*)arg;
1829 	struct outside_network* outnet = sq->outnet;
1830 	struct timeval now = *sq->outnet->now_tv;
1831 	int fallback_tcp = 0;
1832 
1833 	sq->pending = NULL; /* removed after callback */
1834 	if(error == NETEVENT_TIMEOUT) {
1835 		int rto = 0;
1836 		if(sq->status == serviced_query_PROBE_EDNS) {
1837 			/* non-EDNS probe failed; we do not know its status,
1838 			 * keep trying with EDNS, timeout may not be caused
1839 			 * by EDNS. */
1840 			sq->status = serviced_query_UDP_EDNS;
1841 		}
1842 		if(sq->status == serviced_query_UDP_EDNS && sq->last_rtt < 5000) {
1843 			/* fallback to 1480/1280 */
1844 			sq->status = serviced_query_UDP_EDNS_FRAG;
1845 			log_name_addr(VERB_ALGO, "try edns1xx0", sq->qbuf+10,
1846 				&sq->addr, sq->addrlen);
1847 			if(!serviced_udp_send(sq, c->buffer)) {
1848 				serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1849 			}
1850 			return 0;
1851 		}
1852 		if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1853 			/* fragmentation size did not fix it */
1854 			sq->status = serviced_query_UDP_EDNS;
1855 		}
1856 		sq->retry++;
1857 		if(!(rto=infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
1858 			sq->zone, sq->zonelen, sq->qtype, -1, sq->last_rtt,
1859 			(time_t)now.tv_sec)))
1860 			log_err("out of memory in UDP exponential backoff");
1861 		if(sq->retry < OUTBOUND_UDP_RETRY) {
1862 			log_name_addr(VERB_ALGO, "retry query", sq->qbuf+10,
1863 				&sq->addr, sq->addrlen);
1864 			if(!serviced_udp_send(sq, c->buffer)) {
1865 				serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1866 			}
1867 			return 0;
1868 		}
1869 		if(rto >= RTT_MAX_TIMEOUT) {
1870 			fallback_tcp = 1;
1871 			/* UDP does not work, fallback to TCP below */
1872 		} else {
1873 			serviced_callbacks(sq, NETEVENT_TIMEOUT, c, rep);
1874 			return 0;
1875 		}
1876 	} else if(error != NETEVENT_NOERROR) {
1877 		/* udp returns error (due to no ID or interface available) */
1878 		serviced_callbacks(sq, error, c, rep);
1879 		return 0;
1880 	}
1881 #ifdef USE_DNSTAP
1882 	if(outnet->dtenv &&
1883 	   (outnet->dtenv->log_resolver_response_messages ||
1884 	    outnet->dtenv->log_forwarder_response_messages))
1885 		dt_msg_send_outside_response(outnet->dtenv, &sq->addr, c->type,
1886 		sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
1887 		&sq->last_sent_time, sq->outnet->now_tv, c->buffer);
1888 #endif
1889 	if(!fallback_tcp) {
1890 	    if( (sq->status == serviced_query_UDP_EDNS
1891 	        ||sq->status == serviced_query_UDP_EDNS_FRAG)
1892 		&& (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1893 			== LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(
1894 			sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOTIMPL
1895 		    || packet_edns_malformed(c->buffer, sq->qtype)
1896 			)) {
1897 		/* try to get an answer by falling back without EDNS */
1898 		verbose(VERB_ALGO, "serviced query: attempt without EDNS");
1899 		sq->status = serviced_query_UDP_EDNS_fallback;
1900 		sq->retry = 0;
1901 		if(!serviced_udp_send(sq, c->buffer)) {
1902 			serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1903 		}
1904 		return 0;
1905 	    } else if(sq->status == serviced_query_PROBE_EDNS) {
1906 		/* probe without EDNS succeeds, so we conclude that this
1907 		 * host likely has EDNS packets dropped */
1908 		log_addr(VERB_DETAIL, "timeouts, concluded that connection to "
1909 			"host drops EDNS packets", &sq->addr, sq->addrlen);
1910 		/* only store noEDNS in cache if domain is noDNSSEC */
1911 		if(!sq->want_dnssec)
1912 		  if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1913 			sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
1914 			log_err("Out of memory caching no edns for host");
1915 		  }
1916 		sq->status = serviced_query_UDP;
1917 	    } else if(sq->status == serviced_query_UDP_EDNS &&
1918 		!sq->edns_lame_known) {
1919 		/* now we know that edns queries received answers store that */
1920 		log_addr(VERB_ALGO, "serviced query: EDNS works for",
1921 			&sq->addr, sq->addrlen);
1922 		if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1923 			sq->zone, sq->zonelen, 0, (time_t)now.tv_sec)) {
1924 			log_err("Out of memory caching edns works");
1925 		}
1926 		sq->edns_lame_known = 1;
1927 	    } else if(sq->status == serviced_query_UDP_EDNS_fallback &&
1928 		!sq->edns_lame_known && (LDNS_RCODE_WIRE(
1929 		sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOERROR ||
1930 		LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1931 		LDNS_RCODE_NXDOMAIN || LDNS_RCODE_WIRE(sldns_buffer_begin(
1932 		c->buffer)) == LDNS_RCODE_YXDOMAIN)) {
1933 		/* the fallback produced a result that looks promising, note
1934 		 * that this server should be approached without EDNS */
1935 		/* only store noEDNS in cache if domain is noDNSSEC */
1936 		if(!sq->want_dnssec) {
1937 		  log_addr(VERB_ALGO, "serviced query: EDNS fails for",
1938 			&sq->addr, sq->addrlen);
1939 		  if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1940 			sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
1941 			log_err("Out of memory caching no edns for host");
1942 		  }
1943 		} else {
1944 		  log_addr(VERB_ALGO, "serviced query: EDNS fails, but "
1945 		  	"not stored because need DNSSEC for", &sq->addr,
1946 			sq->addrlen);
1947 		}
1948 		sq->status = serviced_query_UDP;
1949 	    }
1950 	    if(now.tv_sec > sq->last_sent_time.tv_sec ||
1951 		(now.tv_sec == sq->last_sent_time.tv_sec &&
1952 		now.tv_usec > sq->last_sent_time.tv_usec)) {
1953 		/* convert from microseconds to milliseconds */
1954 		int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
1955 		  + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
1956 		verbose(VERB_ALGO, "measured roundtrip at %d msec", roundtime);
1957 		log_assert(roundtime >= 0);
1958 		/* in case the system hibernated, do not enter a huge value,
1959 		 * above this value gives trouble with server selection */
1960 		if(roundtime < 60000) {
1961 		    if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
1962 			sq->zone, sq->zonelen, sq->qtype, roundtime,
1963 			sq->last_rtt, (time_t)now.tv_sec))
1964 			log_err("out of memory noting rtt.");
1965 		}
1966 	    }
1967 	} /* end of if_!fallback_tcp */
1968 	/* perform TC flag check and TCP fallback after updating our
1969 	 * cache entries for EDNS status and RTT times */
1970 	if(LDNS_TC_WIRE(sldns_buffer_begin(c->buffer)) || fallback_tcp) {
1971 		/* fallback to TCP */
1972 		/* this discards partial UDP contents */
1973 		if(sq->status == serviced_query_UDP_EDNS ||
1974 			sq->status == serviced_query_UDP_EDNS_FRAG ||
1975 			sq->status == serviced_query_UDP_EDNS_fallback)
1976 			/* if we have unfinished EDNS_fallback, start again */
1977 			sq->status = serviced_query_TCP_EDNS;
1978 		else	sq->status = serviced_query_TCP;
1979 		serviced_tcp_initiate(sq, c->buffer);
1980 		return 0;
1981 	}
1982 	/* yay! an answer */
1983 	serviced_callbacks(sq, error, c, rep);
1984 	return 0;
1985 }
1986 
1987 struct serviced_query*
1988 outnet_serviced_query(struct outside_network* outnet,
1989 	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
1990 	uint16_t flags, int dnssec, int want_dnssec, int nocaps,
1991 	int tcp_upstream, int ssl_upstream, struct edns_option* opt_list,
1992 	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
1993 	size_t zonelen, comm_point_callback_t* callback, void* callback_arg,
1994 	sldns_buffer* buff)
1995 {
1996 	struct serviced_query* sq;
1997 	struct service_callback* cb;
1998 	serviced_gen_query(buff, qname, qnamelen, qtype, qclass, flags);
1999 	sq = lookup_serviced(outnet, buff, dnssec, addr, addrlen, opt_list);
2000 	/* duplicate entries are included in the callback list, because
2001 	 * there is a counterpart registration by our caller that needs to
2002 	 * be doubly-removed (with callbacks perhaps). */
2003 	if(!(cb = (struct service_callback*)malloc(sizeof(*cb))))
2004 		return NULL;
2005 	if(!sq) {
2006 		/* make new serviced query entry */
2007 		sq = serviced_create(outnet, buff, dnssec, want_dnssec, nocaps,
2008 			tcp_upstream, ssl_upstream, addr, addrlen, zone,
2009 			zonelen, (int)qtype, opt_list);
2010 		if(!sq) {
2011 			free(cb);
2012 			return NULL;
2013 		}
2014 		/* perform first network action */
2015 		if(outnet->do_udp && !(tcp_upstream || ssl_upstream)) {
2016 			if(!serviced_udp_send(sq, buff)) {
2017 				(void)rbtree_delete(outnet->serviced, sq);
2018 				free(sq->qbuf);
2019 				free(sq->zone);
2020 				free(sq);
2021 				free(cb);
2022 				return NULL;
2023 			}
2024 		} else {
2025 			if(!serviced_tcp_send(sq, buff)) {
2026 				(void)rbtree_delete(outnet->serviced, sq);
2027 				free(sq->qbuf);
2028 				free(sq->zone);
2029 				free(sq);
2030 				free(cb);
2031 				return NULL;
2032 			}
2033 		}
2034 	}
2035 	/* add callback to list of callbacks */
2036 	cb->cb = callback;
2037 	cb->cb_arg = callback_arg;
2038 	cb->next = sq->cblist;
2039 	sq->cblist = cb;
2040 	return sq;
2041 }
2042 
2043 /** remove callback from list */
2044 static void
2045 callback_list_remove(struct serviced_query* sq, void* cb_arg)
2046 {
2047 	struct service_callback** pp = &sq->cblist;
2048 	while(*pp) {
2049 		if((*pp)->cb_arg == cb_arg) {
2050 			struct service_callback* del = *pp;
2051 			*pp = del->next;
2052 			free(del);
2053 			return;
2054 		}
2055 		pp = &(*pp)->next;
2056 	}
2057 }
2058 
2059 void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
2060 {
2061 	if(!sq)
2062 		return;
2063 	callback_list_remove(sq, cb_arg);
2064 	/* if callbacks() routine scheduled deletion, let it do that */
2065 	if(!sq->cblist && !sq->to_be_deleted) {
2066 		(void)rbtree_delete(sq->outnet->serviced, sq);
2067 		serviced_delete(sq);
2068 	}
2069 }
2070 
2071 /** get memory used by waiting tcp entry (in use or not) */
2072 static size_t
2073 waiting_tcp_get_mem(struct waiting_tcp* w)
2074 {
2075 	size_t s;
2076 	if(!w) return 0;
2077 	s = sizeof(*w) + w->pkt_len;
2078 	if(w->timer)
2079 		s += comm_timer_get_mem(w->timer);
2080 	return s;
2081 }
2082 
2083 /** get memory used by port if */
2084 static size_t
2085 if_get_mem(struct port_if* pif)
2086 {
2087 	size_t s;
2088 	int i;
2089 	s = sizeof(*pif) + sizeof(int)*pif->avail_total +
2090 		sizeof(struct port_comm*)*pif->maxout;
2091 	for(i=0; i<pif->inuse; i++)
2092 		s += sizeof(*pif->out[i]) +
2093 			comm_point_get_mem(pif->out[i]->cp);
2094 	return s;
2095 }
2096 
2097 /** get memory used by waiting udp */
2098 static size_t
2099 waiting_udp_get_mem(struct pending* w)
2100 {
2101 	size_t s;
2102 	s = sizeof(*w) + comm_timer_get_mem(w->timer) + w->pkt_len;
2103 	return s;
2104 }
2105 
2106 size_t outnet_get_mem(struct outside_network* outnet)
2107 {
2108 	size_t i;
2109 	int k;
2110 	struct waiting_tcp* w;
2111 	struct pending* u;
2112 	struct serviced_query* sq;
2113 	struct service_callback* sb;
2114 	struct port_comm* pc;
2115 	size_t s = sizeof(*outnet) + sizeof(*outnet->base) +
2116 		sizeof(*outnet->udp_buff) +
2117 		sldns_buffer_capacity(outnet->udp_buff);
2118 	/* second buffer is not ours */
2119 	for(pc = outnet->unused_fds; pc; pc = pc->next) {
2120 		s += sizeof(*pc) + comm_point_get_mem(pc->cp);
2121 	}
2122 	for(k=0; k<outnet->num_ip4; k++)
2123 		s += if_get_mem(&outnet->ip4_ifs[k]);
2124 	for(k=0; k<outnet->num_ip6; k++)
2125 		s += if_get_mem(&outnet->ip6_ifs[k]);
2126 	for(u=outnet->udp_wait_first; u; u=u->next_waiting)
2127 		s += waiting_udp_get_mem(u);
2128 
2129 	s += sizeof(struct pending_tcp*)*outnet->num_tcp;
2130 	for(i=0; i<outnet->num_tcp; i++) {
2131 		s += sizeof(struct pending_tcp);
2132 		s += comm_point_get_mem(outnet->tcp_conns[i]->c);
2133 		if(outnet->tcp_conns[i]->query)
2134 			s += waiting_tcp_get_mem(outnet->tcp_conns[i]->query);
2135 	}
2136 	for(w=outnet->tcp_wait_first; w; w = w->next_waiting)
2137 		s += waiting_tcp_get_mem(w);
2138 	s += sizeof(*outnet->pending);
2139 	s += (sizeof(struct pending) + comm_timer_get_mem(NULL)) *
2140 		outnet->pending->count;
2141 	s += sizeof(*outnet->serviced);
2142 	s += outnet->svcd_overhead;
2143 	RBTREE_FOR(sq, struct serviced_query*, outnet->serviced) {
2144 		s += sizeof(*sq) + sq->qbuflen;
2145 		for(sb = sq->cblist; sb; sb = sb->next)
2146 			s += sizeof(*sb);
2147 	}
2148 	return s;
2149 }
2150 
2151 size_t
2152 serviced_get_mem(struct serviced_query* sq)
2153 {
2154 	struct service_callback* sb;
2155 	size_t s;
2156 	s = sizeof(*sq) + sq->qbuflen;
2157 	for(sb = sq->cblist; sb; sb = sb->next)
2158 		s += sizeof(*sb);
2159 	if(sq->status == serviced_query_UDP_EDNS ||
2160 		sq->status == serviced_query_UDP ||
2161 		sq->status == serviced_query_PROBE_EDNS ||
2162 		sq->status == serviced_query_UDP_EDNS_FRAG ||
2163 		sq->status == serviced_query_UDP_EDNS_fallback) {
2164 		s += sizeof(struct pending);
2165 		s += comm_timer_get_mem(NULL);
2166 	} else {
2167 		/* does not have size of the pkt pointer */
2168 		/* always has a timer except on malloc failures */
2169 
2170 		/* these sizes are part of the main outside network mem */
2171 		/*
2172 		s += sizeof(struct waiting_tcp);
2173 		s += comm_timer_get_mem(NULL);
2174 		*/
2175 	}
2176 	return s;
2177 }
2178 
2179