xref: /freebsd/contrib/unbound/util/netevent.c (revision f976241773df2260e6170317080761d1c5814fe5)
1 /*
2  * util/netevent.c - event notification
3  *
4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
5  *
6  * This software is open source.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright notice,
13  * this list of conditions and the following disclaimer.
14  *
15  * Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  *
19  * Neither the name of the NLNET LABS nor the names of its contributors may
20  * be used to endorse or promote products derived from this software without
21  * specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 /**
37  * \file
38  *
39  * This file contains event notification functions.
40  */
41 #include "config.h"
42 #include "util/netevent.h"
43 #include "util/ub_event.h"
44 #include "util/log.h"
45 #include "util/net_help.h"
46 #include "util/tcp_conn_limit.h"
47 #include "util/fptr_wlist.h"
48 #include "sldns/pkthdr.h"
49 #include "sldns/sbuffer.h"
50 #include "sldns/str2wire.h"
51 #include "dnstap/dnstap.h"
52 #include "dnscrypt/dnscrypt.h"
53 #include "services/listen_dnsport.h"
54 #ifdef HAVE_OPENSSL_SSL_H
55 #include <openssl/ssl.h>
56 #endif
57 #ifdef HAVE_OPENSSL_ERR_H
58 #include <openssl/err.h>
59 #endif
60 
61 /* -------- Start of local definitions -------- */
62 /** if CMSG_ALIGN is not defined on this platform, a workaround */
63 #ifndef CMSG_ALIGN
64 #  ifdef __CMSG_ALIGN
65 #    define CMSG_ALIGN(n) __CMSG_ALIGN(n)
66 #  elif defined(CMSG_DATA_ALIGN)
67 #    define CMSG_ALIGN _CMSG_DATA_ALIGN
68 #  else
69 #    define CMSG_ALIGN(len) (((len)+sizeof(long)-1) & ~(sizeof(long)-1))
70 #  endif
71 #endif
72 
73 /** if CMSG_LEN is not defined on this platform, a workaround */
74 #ifndef CMSG_LEN
75 #  define CMSG_LEN(len) (CMSG_ALIGN(sizeof(struct cmsghdr))+(len))
76 #endif
77 
78 /** if CMSG_SPACE is not defined on this platform, a workaround */
79 #ifndef CMSG_SPACE
80 #  ifdef _CMSG_HDR_ALIGN
81 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+_CMSG_HDR_ALIGN(sizeof(struct cmsghdr)))
82 #  else
83 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+CMSG_ALIGN(sizeof(struct cmsghdr)))
84 #  endif
85 #endif
86 
87 /** The TCP writing query timeout in milliseconds */
88 #define TCP_QUERY_TIMEOUT 120000
89 /** The minimum actual TCP timeout to use, regardless of what we advertise,
90  * in msec */
91 #define TCP_QUERY_TIMEOUT_MINIMUM 200
92 
93 #ifndef NONBLOCKING_IS_BROKEN
94 /** number of UDP reads to perform per read indication from select */
95 #define NUM_UDP_PER_SELECT 100
96 #else
97 #define NUM_UDP_PER_SELECT 1
98 #endif
99 
100 /**
101  * The internal event structure for keeping ub_event info for the event.
102  * Possibly other structures (list, tree) this is part of.
103  */
104 struct internal_event {
105 	/** the comm base */
106 	struct comm_base* base;
107 	/** ub_event event type */
108 	struct ub_event* ev;
109 };
110 
111 /**
112  * Internal base structure, so that every thread has its own events.
113  */
114 struct internal_base {
115 	/** ub_event event_base type. */
116 	struct ub_event_base* base;
117 	/** seconds time pointer points here */
118 	time_t secs;
119 	/** timeval with current time */
120 	struct timeval now;
121 	/** the event used for slow_accept timeouts */
122 	struct ub_event* slow_accept;
123 	/** true if slow_accept is enabled */
124 	int slow_accept_enabled;
125 };
126 
127 /**
128  * Internal timer structure, to store timer event in.
129  */
130 struct internal_timer {
131 	/** the super struct from which derived */
132 	struct comm_timer super;
133 	/** the comm base */
134 	struct comm_base* base;
135 	/** ub_event event type */
136 	struct ub_event* ev;
137 	/** is timer enabled */
138 	uint8_t enabled;
139 };
140 
141 /**
142  * Internal signal structure, to store signal event in.
143  */
144 struct internal_signal {
145 	/** ub_event event type */
146 	struct ub_event* ev;
147 	/** next in signal list */
148 	struct internal_signal* next;
149 };
150 
151 /** create a tcp handler with a parent */
152 static struct comm_point* comm_point_create_tcp_handler(
153 	struct comm_base *base, struct comm_point* parent, size_t bufsize,
154 	struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
155 	void* callback_arg);
156 
157 /* -------- End of local definitions -------- */
158 
159 struct comm_base*
160 comm_base_create(int sigs)
161 {
162 	struct comm_base* b = (struct comm_base*)calloc(1,
163 		sizeof(struct comm_base));
164 	const char *evnm="event", *evsys="", *evmethod="";
165 
166 	if(!b)
167 		return NULL;
168 	b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
169 	if(!b->eb) {
170 		free(b);
171 		return NULL;
172 	}
173 	b->eb->base = ub_default_event_base(sigs, &b->eb->secs, &b->eb->now);
174 	if(!b->eb->base) {
175 		free(b->eb);
176 		free(b);
177 		return NULL;
178 	}
179 	ub_comm_base_now(b);
180 	ub_get_event_sys(b->eb->base, &evnm, &evsys, &evmethod);
181 	verbose(VERB_ALGO, "%s %s uses %s method.", evnm, evsys, evmethod);
182 	return b;
183 }
184 
185 struct comm_base*
186 comm_base_create_event(struct ub_event_base* base)
187 {
188 	struct comm_base* b = (struct comm_base*)calloc(1,
189 		sizeof(struct comm_base));
190 	if(!b)
191 		return NULL;
192 	b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
193 	if(!b->eb) {
194 		free(b);
195 		return NULL;
196 	}
197 	b->eb->base = base;
198 	ub_comm_base_now(b);
199 	return b;
200 }
201 
202 void
203 comm_base_delete(struct comm_base* b)
204 {
205 	if(!b)
206 		return;
207 	if(b->eb->slow_accept_enabled) {
208 		if(ub_event_del(b->eb->slow_accept) != 0) {
209 			log_err("could not event_del slow_accept");
210 		}
211 		ub_event_free(b->eb->slow_accept);
212 	}
213 	ub_event_base_free(b->eb->base);
214 	b->eb->base = NULL;
215 	free(b->eb);
216 	free(b);
217 }
218 
219 void
220 comm_base_delete_no_base(struct comm_base* b)
221 {
222 	if(!b)
223 		return;
224 	if(b->eb->slow_accept_enabled) {
225 		if(ub_event_del(b->eb->slow_accept) != 0) {
226 			log_err("could not event_del slow_accept");
227 		}
228 		ub_event_free(b->eb->slow_accept);
229 	}
230 	b->eb->base = NULL;
231 	free(b->eb);
232 	free(b);
233 }
234 
235 void
236 comm_base_timept(struct comm_base* b, time_t** tt, struct timeval** tv)
237 {
238 	*tt = &b->eb->secs;
239 	*tv = &b->eb->now;
240 }
241 
242 void
243 comm_base_dispatch(struct comm_base* b)
244 {
245 	int retval;
246 	retval = ub_event_base_dispatch(b->eb->base);
247 	if(retval < 0) {
248 		fatal_exit("event_dispatch returned error %d, "
249 			"errno is %s", retval, strerror(errno));
250 	}
251 }
252 
253 void comm_base_exit(struct comm_base* b)
254 {
255 	if(ub_event_base_loopexit(b->eb->base) != 0) {
256 		log_err("Could not loopexit");
257 	}
258 }
259 
260 void comm_base_set_slow_accept_handlers(struct comm_base* b,
261 	void (*stop_acc)(void*), void (*start_acc)(void*), void* arg)
262 {
263 	b->stop_accept = stop_acc;
264 	b->start_accept = start_acc;
265 	b->cb_arg = arg;
266 }
267 
268 struct ub_event_base* comm_base_internal(struct comm_base* b)
269 {
270 	return b->eb->base;
271 }
272 
273 /** see if errno for udp has to be logged or not uses globals */
274 static int
275 udp_send_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
276 {
277 	/* do not log transient errors (unless high verbosity) */
278 #if defined(ENETUNREACH) || defined(EHOSTDOWN) || defined(EHOSTUNREACH) || defined(ENETDOWN)
279 	switch(errno) {
280 #  ifdef ENETUNREACH
281 		case ENETUNREACH:
282 #  endif
283 #  ifdef EHOSTDOWN
284 		case EHOSTDOWN:
285 #  endif
286 #  ifdef EHOSTUNREACH
287 		case EHOSTUNREACH:
288 #  endif
289 #  ifdef ENETDOWN
290 		case ENETDOWN:
291 #  endif
292 			if(verbosity < VERB_ALGO)
293 				return 0;
294 		default:
295 			break;
296 	}
297 #endif
298 	/* permission denied is gotten for every send if the
299 	 * network is disconnected (on some OS), squelch it */
300 	if( ((errno == EPERM)
301 #  ifdef EADDRNOTAVAIL
302 		/* 'Cannot assign requested address' also when disconnected */
303 		|| (errno == EADDRNOTAVAIL)
304 #  endif
305 		) && verbosity < VERB_DETAIL)
306 		return 0;
307 #  ifdef EADDRINUSE
308 	/* If SO_REUSEADDR is set, we could try to connect to the same server
309 	 * from the same source port twice. */
310 	if(errno == EADDRINUSE && verbosity < VERB_DETAIL)
311 		return 0;
312 #  endif
313 	/* squelch errors where people deploy AAAA ::ffff:bla for
314 	 * authority servers, which we try for intranets. */
315 	if(errno == EINVAL && addr_is_ip4mapped(
316 		(struct sockaddr_storage*)addr, addrlen) &&
317 		verbosity < VERB_DETAIL)
318 		return 0;
319 	/* SO_BROADCAST sockopt can give access to 255.255.255.255,
320 	 * but a dns cache does not need it. */
321 	if(errno == EACCES && addr_is_broadcast(
322 		(struct sockaddr_storage*)addr, addrlen) &&
323 		verbosity < VERB_DETAIL)
324 		return 0;
325 	return 1;
326 }
327 
328 int tcp_connect_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
329 {
330 	return udp_send_errno_needs_log(addr, addrlen);
331 }
332 
333 /* send a UDP reply */
334 int
335 comm_point_send_udp_msg(struct comm_point *c, sldns_buffer* packet,
336 	struct sockaddr* addr, socklen_t addrlen)
337 {
338 	ssize_t sent;
339 	log_assert(c->fd != -1);
340 #ifdef UNBOUND_DEBUG
341 	if(sldns_buffer_remaining(packet) == 0)
342 		log_err("error: send empty UDP packet");
343 #endif
344 	log_assert(addr && addrlen > 0);
345 	sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
346 		sldns_buffer_remaining(packet), 0,
347 		addr, addrlen);
348 	if(sent == -1) {
349 		/* try again and block, waiting for IO to complete,
350 		 * we want to send the answer, and we will wait for
351 		 * the ethernet interface buffer to have space. */
352 #ifndef USE_WINSOCK
353 		if(errno == EAGAIN ||
354 #  ifdef EWOULDBLOCK
355 			errno == EWOULDBLOCK ||
356 #  endif
357 			errno == ENOBUFS) {
358 #else
359 		if(WSAGetLastError() == WSAEINPROGRESS ||
360 			WSAGetLastError() == WSAENOBUFS ||
361 			WSAGetLastError() == WSAEWOULDBLOCK) {
362 #endif
363 			int e;
364 			fd_set_block(c->fd);
365 			sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
366 				sldns_buffer_remaining(packet), 0,
367 				addr, addrlen);
368 			e = errno;
369 			fd_set_nonblock(c->fd);
370 			errno = e;
371 		}
372 	}
373 	if(sent == -1) {
374 		if(!udp_send_errno_needs_log(addr, addrlen))
375 			return 0;
376 #ifndef USE_WINSOCK
377 		verbose(VERB_OPS, "sendto failed: %s", strerror(errno));
378 #else
379 		verbose(VERB_OPS, "sendto failed: %s",
380 			wsa_strerror(WSAGetLastError()));
381 #endif
382 		log_addr(VERB_OPS, "remote address is",
383 			(struct sockaddr_storage*)addr, addrlen);
384 		return 0;
385 	} else if((size_t)sent != sldns_buffer_remaining(packet)) {
386 		log_err("sent %d in place of %d bytes",
387 			(int)sent, (int)sldns_buffer_remaining(packet));
388 		return 0;
389 	}
390 	return 1;
391 }
392 
393 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && (defined(HAVE_RECVMSG) || defined(HAVE_SENDMSG))
394 /** print debug ancillary info */
395 static void p_ancil(const char* str, struct comm_reply* r)
396 {
397 	if(r->srctype != 4 && r->srctype != 6) {
398 		log_info("%s: unknown srctype %d", str, r->srctype);
399 		return;
400 	}
401 	if(r->srctype == 6) {
402 		char buf[1024];
403 		if(inet_ntop(AF_INET6, &r->pktinfo.v6info.ipi6_addr,
404 			buf, (socklen_t)sizeof(buf)) == 0) {
405 			(void)strlcpy(buf, "(inet_ntop error)", sizeof(buf));
406 		}
407 		buf[sizeof(buf)-1]=0;
408 		log_info("%s: %s %d", str, buf, r->pktinfo.v6info.ipi6_ifindex);
409 	} else if(r->srctype == 4) {
410 #ifdef IP_PKTINFO
411 		char buf1[1024], buf2[1024];
412 		if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_addr,
413 			buf1, (socklen_t)sizeof(buf1)) == 0) {
414 			(void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
415 		}
416 		buf1[sizeof(buf1)-1]=0;
417 #ifdef HAVE_STRUCT_IN_PKTINFO_IPI_SPEC_DST
418 		if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_spec_dst,
419 			buf2, (socklen_t)sizeof(buf2)) == 0) {
420 			(void)strlcpy(buf2, "(inet_ntop error)", sizeof(buf2));
421 		}
422 		buf2[sizeof(buf2)-1]=0;
423 #else
424 		buf2[0]=0;
425 #endif
426 		log_info("%s: %d %s %s", str, r->pktinfo.v4info.ipi_ifindex,
427 			buf1, buf2);
428 #elif defined(IP_RECVDSTADDR)
429 		char buf1[1024];
430 		if(inet_ntop(AF_INET, &r->pktinfo.v4addr,
431 			buf1, (socklen_t)sizeof(buf1)) == 0) {
432 			(void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
433 		}
434 		buf1[sizeof(buf1)-1]=0;
435 		log_info("%s: %s", str, buf1);
436 #endif /* IP_PKTINFO or PI_RECVDSTDADDR */
437 	}
438 }
439 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG||HAVE_SENDMSG */
440 
441 /** send a UDP reply over specified interface*/
442 static int
443 comm_point_send_udp_msg_if(struct comm_point *c, sldns_buffer* packet,
444 	struct sockaddr* addr, socklen_t addrlen, struct comm_reply* r)
445 {
446 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_SENDMSG)
447 	ssize_t sent;
448 	struct msghdr msg;
449 	struct iovec iov[1];
450 	char control[256];
451 #ifndef S_SPLINT_S
452 	struct cmsghdr *cmsg;
453 #endif /* S_SPLINT_S */
454 
455 	log_assert(c->fd != -1);
456 #ifdef UNBOUND_DEBUG
457 	if(sldns_buffer_remaining(packet) == 0)
458 		log_err("error: send empty UDP packet");
459 #endif
460 	log_assert(addr && addrlen > 0);
461 
462 	msg.msg_name = addr;
463 	msg.msg_namelen = addrlen;
464 	iov[0].iov_base = sldns_buffer_begin(packet);
465 	iov[0].iov_len = sldns_buffer_remaining(packet);
466 	msg.msg_iov = iov;
467 	msg.msg_iovlen = 1;
468 	msg.msg_control = control;
469 #ifndef S_SPLINT_S
470 	msg.msg_controllen = sizeof(control);
471 #endif /* S_SPLINT_S */
472 	msg.msg_flags = 0;
473 
474 #ifndef S_SPLINT_S
475 	cmsg = CMSG_FIRSTHDR(&msg);
476 	if(r->srctype == 4) {
477 #ifdef IP_PKTINFO
478 		void* cmsg_data;
479 		msg.msg_controllen = CMSG_SPACE(sizeof(struct in_pktinfo));
480 		log_assert(msg.msg_controllen <= sizeof(control));
481 		cmsg->cmsg_level = IPPROTO_IP;
482 		cmsg->cmsg_type = IP_PKTINFO;
483 		memmove(CMSG_DATA(cmsg), &r->pktinfo.v4info,
484 			sizeof(struct in_pktinfo));
485 		/* unset the ifindex to not bypass the routing tables */
486 		cmsg_data = CMSG_DATA(cmsg);
487 		((struct in_pktinfo *) cmsg_data)->ipi_ifindex = 0;
488 		cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo));
489 #elif defined(IP_SENDSRCADDR)
490 		msg.msg_controllen = CMSG_SPACE(sizeof(struct in_addr));
491 		log_assert(msg.msg_controllen <= sizeof(control));
492 		cmsg->cmsg_level = IPPROTO_IP;
493 		cmsg->cmsg_type = IP_SENDSRCADDR;
494 		memmove(CMSG_DATA(cmsg), &r->pktinfo.v4addr,
495 			sizeof(struct in_addr));
496 		cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_addr));
497 #else
498 		verbose(VERB_ALGO, "no IP_PKTINFO or IP_SENDSRCADDR");
499 		msg.msg_control = NULL;
500 #endif /* IP_PKTINFO or IP_SENDSRCADDR */
501 	} else if(r->srctype == 6) {
502 		void* cmsg_data;
503 		msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
504 		log_assert(msg.msg_controllen <= sizeof(control));
505 		cmsg->cmsg_level = IPPROTO_IPV6;
506 		cmsg->cmsg_type = IPV6_PKTINFO;
507 		memmove(CMSG_DATA(cmsg), &r->pktinfo.v6info,
508 			sizeof(struct in6_pktinfo));
509 		/* unset the ifindex to not bypass the routing tables */
510 		cmsg_data = CMSG_DATA(cmsg);
511 		((struct in6_pktinfo *) cmsg_data)->ipi6_ifindex = 0;
512 		cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
513 	} else {
514 		/* try to pass all 0 to use default route */
515 		msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
516 		log_assert(msg.msg_controllen <= sizeof(control));
517 		cmsg->cmsg_level = IPPROTO_IPV6;
518 		cmsg->cmsg_type = IPV6_PKTINFO;
519 		memset(CMSG_DATA(cmsg), 0, sizeof(struct in6_pktinfo));
520 		cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
521 	}
522 #endif /* S_SPLINT_S */
523 	if(verbosity >= VERB_ALGO)
524 		p_ancil("send_udp over interface", r);
525 	sent = sendmsg(c->fd, &msg, 0);
526 	if(sent == -1) {
527 		/* try again and block, waiting for IO to complete,
528 		 * we want to send the answer, and we will wait for
529 		 * the ethernet interface buffer to have space. */
530 #ifndef USE_WINSOCK
531 		if(errno == EAGAIN ||
532 #  ifdef EWOULDBLOCK
533 			errno == EWOULDBLOCK ||
534 #  endif
535 			errno == ENOBUFS) {
536 #else
537 		if(WSAGetLastError() == WSAEINPROGRESS ||
538 			WSAGetLastError() == WSAENOBUFS ||
539 			WSAGetLastError() == WSAEWOULDBLOCK) {
540 #endif
541 			int e;
542 			fd_set_block(c->fd);
543 			sent = sendmsg(c->fd, &msg, 0);
544 			e = errno;
545 			fd_set_nonblock(c->fd);
546 			errno = e;
547 		}
548 	}
549 	if(sent == -1) {
550 		if(!udp_send_errno_needs_log(addr, addrlen))
551 			return 0;
552 		verbose(VERB_OPS, "sendmsg failed: %s", strerror(errno));
553 		log_addr(VERB_OPS, "remote address is",
554 			(struct sockaddr_storage*)addr, addrlen);
555 #ifdef __NetBSD__
556 		/* netbsd 7 has IP_PKTINFO for recv but not send */
557 		if(errno == EINVAL && r->srctype == 4)
558 			log_err("sendmsg: No support for sendmsg(IP_PKTINFO). "
559 				"Please disable interface-automatic");
560 #endif
561 		return 0;
562 	} else if((size_t)sent != sldns_buffer_remaining(packet)) {
563 		log_err("sent %d in place of %d bytes",
564 			(int)sent, (int)sldns_buffer_remaining(packet));
565 		return 0;
566 	}
567 	return 1;
568 #else
569 	(void)c;
570 	(void)packet;
571 	(void)addr;
572 	(void)addrlen;
573 	(void)r;
574 	log_err("sendmsg: IPV6_PKTINFO not supported");
575 	return 0;
576 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_SENDMSG */
577 }
578 
579 void
580 comm_point_udp_ancil_callback(int fd, short event, void* arg)
581 {
582 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_RECVMSG)
583 	struct comm_reply rep;
584 	struct msghdr msg;
585 	struct iovec iov[1];
586 	ssize_t rcv;
587 	char ancil[256];
588 	int i;
589 #ifndef S_SPLINT_S
590 	struct cmsghdr* cmsg;
591 #endif /* S_SPLINT_S */
592 
593 	rep.c = (struct comm_point*)arg;
594 	log_assert(rep.c->type == comm_udp);
595 
596 	if(!(event&UB_EV_READ))
597 		return;
598 	log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
599 	ub_comm_base_now(rep.c->ev->base);
600 	for(i=0; i<NUM_UDP_PER_SELECT; i++) {
601 		sldns_buffer_clear(rep.c->buffer);
602 		rep.addrlen = (socklen_t)sizeof(rep.addr);
603 		log_assert(fd != -1);
604 		log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
605 		msg.msg_name = &rep.addr;
606 		msg.msg_namelen = (socklen_t)sizeof(rep.addr);
607 		iov[0].iov_base = sldns_buffer_begin(rep.c->buffer);
608 		iov[0].iov_len = sldns_buffer_remaining(rep.c->buffer);
609 		msg.msg_iov = iov;
610 		msg.msg_iovlen = 1;
611 		msg.msg_control = ancil;
612 #ifndef S_SPLINT_S
613 		msg.msg_controllen = sizeof(ancil);
614 #endif /* S_SPLINT_S */
615 		msg.msg_flags = 0;
616 		rcv = recvmsg(fd, &msg, 0);
617 		if(rcv == -1) {
618 			if(errno != EAGAIN && errno != EINTR) {
619 				log_err("recvmsg failed: %s", strerror(errno));
620 			}
621 			return;
622 		}
623 		rep.addrlen = msg.msg_namelen;
624 		sldns_buffer_skip(rep.c->buffer, rcv);
625 		sldns_buffer_flip(rep.c->buffer);
626 		rep.srctype = 0;
627 #ifndef S_SPLINT_S
628 		for(cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
629 			cmsg = CMSG_NXTHDR(&msg, cmsg)) {
630 			if( cmsg->cmsg_level == IPPROTO_IPV6 &&
631 				cmsg->cmsg_type == IPV6_PKTINFO) {
632 				rep.srctype = 6;
633 				memmove(&rep.pktinfo.v6info, CMSG_DATA(cmsg),
634 					sizeof(struct in6_pktinfo));
635 				break;
636 #ifdef IP_PKTINFO
637 			} else if( cmsg->cmsg_level == IPPROTO_IP &&
638 				cmsg->cmsg_type == IP_PKTINFO) {
639 				rep.srctype = 4;
640 				memmove(&rep.pktinfo.v4info, CMSG_DATA(cmsg),
641 					sizeof(struct in_pktinfo));
642 				break;
643 #elif defined(IP_RECVDSTADDR)
644 			} else if( cmsg->cmsg_level == IPPROTO_IP &&
645 				cmsg->cmsg_type == IP_RECVDSTADDR) {
646 				rep.srctype = 4;
647 				memmove(&rep.pktinfo.v4addr, CMSG_DATA(cmsg),
648 					sizeof(struct in_addr));
649 				break;
650 #endif /* IP_PKTINFO or IP_RECVDSTADDR */
651 			}
652 		}
653 		if(verbosity >= VERB_ALGO)
654 			p_ancil("receive_udp on interface", &rep);
655 #endif /* S_SPLINT_S */
656 		fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
657 		if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
658 			/* send back immediate reply */
659 			(void)comm_point_send_udp_msg_if(rep.c, rep.c->buffer,
660 				(struct sockaddr*)&rep.addr, rep.addrlen, &rep);
661 		}
662 		if(!rep.c || rep.c->fd == -1) /* commpoint closed */
663 			break;
664 	}
665 #else
666 	(void)fd;
667 	(void)event;
668 	(void)arg;
669 	fatal_exit("recvmsg: No support for IPV6_PKTINFO; IP_PKTINFO or IP_RECVDSTADDR. "
670 		"Please disable interface-automatic");
671 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG */
672 }
673 
674 void
675 comm_point_udp_callback(int fd, short event, void* arg)
676 {
677 	struct comm_reply rep;
678 	ssize_t rcv;
679 	int i;
680 	struct sldns_buffer *buffer;
681 
682 	rep.c = (struct comm_point*)arg;
683 	log_assert(rep.c->type == comm_udp);
684 
685 	if(!(event&UB_EV_READ))
686 		return;
687 	log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
688 	ub_comm_base_now(rep.c->ev->base);
689 	for(i=0; i<NUM_UDP_PER_SELECT; i++) {
690 		sldns_buffer_clear(rep.c->buffer);
691 		rep.addrlen = (socklen_t)sizeof(rep.addr);
692 		log_assert(fd != -1);
693 		log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
694 		rcv = recvfrom(fd, (void*)sldns_buffer_begin(rep.c->buffer),
695 			sldns_buffer_remaining(rep.c->buffer), 0,
696 			(struct sockaddr*)&rep.addr, &rep.addrlen);
697 		if(rcv == -1) {
698 #ifndef USE_WINSOCK
699 			if(errno != EAGAIN && errno != EINTR)
700 				log_err("recvfrom %d failed: %s",
701 					fd, strerror(errno));
702 #else
703 			if(WSAGetLastError() != WSAEINPROGRESS &&
704 				WSAGetLastError() != WSAECONNRESET &&
705 				WSAGetLastError()!= WSAEWOULDBLOCK)
706 				log_err("recvfrom failed: %s",
707 					wsa_strerror(WSAGetLastError()));
708 #endif
709 			return;
710 		}
711 		sldns_buffer_skip(rep.c->buffer, rcv);
712 		sldns_buffer_flip(rep.c->buffer);
713 		rep.srctype = 0;
714 		fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
715 		if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
716 			/* send back immediate reply */
717 #ifdef USE_DNSCRYPT
718 			buffer = rep.c->dnscrypt_buffer;
719 #else
720 			buffer = rep.c->buffer;
721 #endif
722 			(void)comm_point_send_udp_msg(rep.c, buffer,
723 				(struct sockaddr*)&rep.addr, rep.addrlen);
724 		}
725 		if(!rep.c || rep.c->fd != fd) /* commpoint closed to -1 or reused for
726 		another UDP port. Note rep.c cannot be reused with TCP fd. */
727 			break;
728 	}
729 }
730 
731 /** Use a new tcp handler for new query fd, set to read query */
732 static void
733 setup_tcp_handler(struct comm_point* c, int fd, int cur, int max)
734 {
735 	int handler_usage;
736 	log_assert(c->type == comm_tcp);
737 	log_assert(c->fd == -1);
738 	sldns_buffer_clear(c->buffer);
739 #ifdef USE_DNSCRYPT
740 	if (c->dnscrypt)
741 		sldns_buffer_clear(c->dnscrypt_buffer);
742 #endif
743 	c->tcp_is_reading = 1;
744 	c->tcp_byte_count = 0;
745 	/* if more than half the tcp handlers are in use, use a shorter
746 	 * timeout for this TCP connection, we need to make space for
747 	 * other connections to be able to get attention */
748 	/* If > 50% TCP handler structures in use, set timeout to 1/100th
749 	 * 	configured value.
750 	 * If > 65%TCP handler structures in use, set to 1/500th configured
751 	 * 	value.
752 	 * If > 80% TCP handler structures in use, set to 0.
753 	 *
754 	 * If the timeout to use falls below 200 milliseconds, an actual
755 	 * timeout of 200ms is used.
756 	 */
757 	handler_usage = (cur * 100) / max;
758 	if(handler_usage > 50 && handler_usage <= 65)
759 		c->tcp_timeout_msec /= 100;
760 	else if (handler_usage > 65 && handler_usage <= 80)
761 		c->tcp_timeout_msec /= 500;
762 	else if (handler_usage > 80)
763 		c->tcp_timeout_msec = 0;
764 	comm_point_start_listening(c, fd,
765 		c->tcp_timeout_msec < TCP_QUERY_TIMEOUT_MINIMUM
766 			? TCP_QUERY_TIMEOUT_MINIMUM
767 			: c->tcp_timeout_msec);
768 }
769 
770 void comm_base_handle_slow_accept(int ATTR_UNUSED(fd),
771 	short ATTR_UNUSED(event), void* arg)
772 {
773 	struct comm_base* b = (struct comm_base*)arg;
774 	/* timeout for the slow accept, re-enable accepts again */
775 	if(b->start_accept) {
776 		verbose(VERB_ALGO, "wait is over, slow accept disabled");
777 		fptr_ok(fptr_whitelist_start_accept(b->start_accept));
778 		(*b->start_accept)(b->cb_arg);
779 		b->eb->slow_accept_enabled = 0;
780 	}
781 }
782 
783 int comm_point_perform_accept(struct comm_point* c,
784 	struct sockaddr_storage* addr, socklen_t* addrlen)
785 {
786 	int new_fd;
787 	*addrlen = (socklen_t)sizeof(*addr);
788 #ifndef HAVE_ACCEPT4
789 	new_fd = accept(c->fd, (struct sockaddr*)addr, addrlen);
790 #else
791 	/* SOCK_NONBLOCK saves extra calls to fcntl for the same result */
792 	new_fd = accept4(c->fd, (struct sockaddr*)addr, addrlen, SOCK_NONBLOCK);
793 #endif
794 	if(new_fd == -1) {
795 #ifndef USE_WINSOCK
796 		/* EINTR is signal interrupt. others are closed connection. */
797 		if(	errno == EINTR || errno == EAGAIN
798 #ifdef EWOULDBLOCK
799 			|| errno == EWOULDBLOCK
800 #endif
801 #ifdef ECONNABORTED
802 			|| errno == ECONNABORTED
803 #endif
804 #ifdef EPROTO
805 			|| errno == EPROTO
806 #endif /* EPROTO */
807 			)
808 			return -1;
809 #if defined(ENFILE) && defined(EMFILE)
810 		if(errno == ENFILE || errno == EMFILE) {
811 			/* out of file descriptors, likely outside of our
812 			 * control. stop accept() calls for some time */
813 			if(c->ev->base->stop_accept) {
814 				struct comm_base* b = c->ev->base;
815 				struct timeval tv;
816 				verbose(VERB_ALGO, "out of file descriptors: "
817 					"slow accept");
818 				b->eb->slow_accept_enabled = 1;
819 				fptr_ok(fptr_whitelist_stop_accept(
820 					b->stop_accept));
821 				(*b->stop_accept)(b->cb_arg);
822 				/* set timeout, no mallocs */
823 				tv.tv_sec = NETEVENT_SLOW_ACCEPT_TIME/1000;
824 				tv.tv_usec = (NETEVENT_SLOW_ACCEPT_TIME%1000)*1000;
825 				b->eb->slow_accept = ub_event_new(b->eb->base,
826 					-1, UB_EV_TIMEOUT,
827 					comm_base_handle_slow_accept, b);
828 				if(b->eb->slow_accept == NULL) {
829 					/* we do not want to log here, because
830 					 * that would spam the logfiles.
831 					 * error: "event_base_set failed." */
832 				}
833 				else if(ub_event_add(b->eb->slow_accept, &tv)
834 					!= 0) {
835 					/* we do not want to log here,
836 					 * error: "event_add failed." */
837 				}
838 			}
839 			return -1;
840 		}
841 #endif
842 		log_err_addr("accept failed", strerror(errno), addr, *addrlen);
843 #else /* USE_WINSOCK */
844 		if(WSAGetLastError() == WSAEINPROGRESS ||
845 			WSAGetLastError() == WSAECONNRESET)
846 			return -1;
847 		if(WSAGetLastError() == WSAEWOULDBLOCK) {
848 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
849 			return -1;
850 		}
851 		log_err_addr("accept failed", wsa_strerror(WSAGetLastError()),
852 			addr, *addrlen);
853 #endif
854 		return -1;
855 	}
856 	if(c->tcp_conn_limit && c->type == comm_tcp_accept) {
857 		c->tcl_addr = tcl_addr_lookup(c->tcp_conn_limit, addr, *addrlen);
858 		if(!tcl_new_connection(c->tcl_addr)) {
859 			if(verbosity >= 3)
860 				log_err_addr("accept rejected",
861 				"connection limit exceeded", addr, *addrlen);
862 			close(new_fd);
863 			return -1;
864 		}
865 	}
866 #ifndef HAVE_ACCEPT4
867 	fd_set_nonblock(new_fd);
868 #endif
869 	return new_fd;
870 }
871 
872 #ifdef USE_WINSOCK
873 static long win_bio_cb(BIO *b, int oper, const char* ATTR_UNUSED(argp),
874         int ATTR_UNUSED(argi), long argl, long retvalue)
875 {
876 	int wsa_err = WSAGetLastError(); /* store errcode before it is gone */
877 	verbose(VERB_ALGO, "bio_cb %d, %s %s %s", oper,
878 		(oper&BIO_CB_RETURN)?"return":"before",
879 		(oper&BIO_CB_READ)?"read":((oper&BIO_CB_WRITE)?"write":"other"),
880 		wsa_err==WSAEWOULDBLOCK?"wsawb":"");
881 	/* on windows, check if previous operation caused EWOULDBLOCK */
882 	if( (oper == (BIO_CB_READ|BIO_CB_RETURN) && argl == 0) ||
883 		(oper == (BIO_CB_GETS|BIO_CB_RETURN) && argl == 0)) {
884 		if(wsa_err == WSAEWOULDBLOCK)
885 			ub_winsock_tcp_wouldblock((struct ub_event*)
886 				BIO_get_callback_arg(b), UB_EV_READ);
887 	}
888 	if( (oper == (BIO_CB_WRITE|BIO_CB_RETURN) && argl == 0) ||
889 		(oper == (BIO_CB_PUTS|BIO_CB_RETURN) && argl == 0)) {
890 		if(wsa_err == WSAEWOULDBLOCK)
891 			ub_winsock_tcp_wouldblock((struct ub_event*)
892 				BIO_get_callback_arg(b), UB_EV_WRITE);
893 	}
894 	/* return original return value */
895 	return retvalue;
896 }
897 
898 /** set win bio callbacks for nonblocking operations */
899 void
900 comm_point_tcp_win_bio_cb(struct comm_point* c, void* thessl)
901 {
902 	SSL* ssl = (SSL*)thessl;
903 	/* set them both just in case, but usually they are the same BIO */
904 	BIO_set_callback(SSL_get_rbio(ssl), &win_bio_cb);
905 	BIO_set_callback_arg(SSL_get_rbio(ssl), (char*)c->ev->ev);
906 	BIO_set_callback(SSL_get_wbio(ssl), &win_bio_cb);
907 	BIO_set_callback_arg(SSL_get_wbio(ssl), (char*)c->ev->ev);
908 }
909 #endif
910 
911 void
912 comm_point_tcp_accept_callback(int fd, short event, void* arg)
913 {
914 	struct comm_point* c = (struct comm_point*)arg, *c_hdl;
915 	int new_fd;
916 	log_assert(c->type == comm_tcp_accept);
917 	if(!(event & UB_EV_READ)) {
918 		log_info("ignoring tcp accept event %d", (int)event);
919 		return;
920 	}
921 	ub_comm_base_now(c->ev->base);
922 	/* find free tcp handler. */
923 	if(!c->tcp_free) {
924 		log_warn("accepted too many tcp, connections full");
925 		return;
926 	}
927 	/* accept incoming connection. */
928 	c_hdl = c->tcp_free;
929 	/* clear leftover flags from previous use, and then set the
930 	 * correct event base for the event structure for libevent */
931 	ub_event_free(c_hdl->ev->ev);
932 	c_hdl->ev->ev = ub_event_new(c_hdl->ev->base->eb->base, -1, UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT, comm_point_tcp_handle_callback, c_hdl);
933 	if(!c_hdl->ev->ev) {
934 		log_warn("could not ub_event_new, dropped tcp");
935 		return;
936 	}
937 	log_assert(fd != -1);
938 	(void)fd;
939 	new_fd = comm_point_perform_accept(c, &c_hdl->repinfo.addr,
940 		&c_hdl->repinfo.addrlen);
941 	if(new_fd == -1)
942 		return;
943 	if(c->ssl) {
944 		c_hdl->ssl = incoming_ssl_fd(c->ssl, new_fd);
945 		if(!c_hdl->ssl) {
946 			c_hdl->fd = new_fd;
947 			comm_point_close(c_hdl);
948 			return;
949 		}
950 		c_hdl->ssl_shake_state = comm_ssl_shake_read;
951 #ifdef USE_WINSOCK
952 		comm_point_tcp_win_bio_cb(c_hdl, c_hdl->ssl);
953 #endif
954 	}
955 
956 	/* grab the tcp handler buffers */
957 	c->cur_tcp_count++;
958 	c->tcp_free = c_hdl->tcp_free;
959 	if(!c->tcp_free) {
960 		/* stop accepting incoming queries for now. */
961 		comm_point_stop_listening(c);
962 	}
963 	setup_tcp_handler(c_hdl, new_fd, c->cur_tcp_count, c->max_tcp_count);
964 }
965 
966 /** Make tcp handler free for next assignment */
967 static void
968 reclaim_tcp_handler(struct comm_point* c)
969 {
970 	log_assert(c->type == comm_tcp);
971 	if(c->ssl) {
972 #ifdef HAVE_SSL
973 		SSL_shutdown(c->ssl);
974 		SSL_free(c->ssl);
975 		c->ssl = NULL;
976 #endif
977 	}
978 	comm_point_close(c);
979 	if(c->tcp_parent) {
980 		c->tcp_parent->cur_tcp_count--;
981 		c->tcp_free = c->tcp_parent->tcp_free;
982 		c->tcp_parent->tcp_free = c;
983 		if(!c->tcp_free) {
984 			/* re-enable listening on accept socket */
985 			comm_point_start_listening(c->tcp_parent, -1, -1);
986 		}
987 	}
988 }
989 
990 /** do the callback when writing is done */
991 static void
992 tcp_callback_writer(struct comm_point* c)
993 {
994 	log_assert(c->type == comm_tcp);
995 	sldns_buffer_clear(c->buffer);
996 	if(c->tcp_do_toggle_rw)
997 		c->tcp_is_reading = 1;
998 	c->tcp_byte_count = 0;
999 	/* switch from listening(write) to listening(read) */
1000 	if(c->tcp_req_info) {
1001 		tcp_req_info_handle_writedone(c->tcp_req_info);
1002 	} else {
1003 		comm_point_stop_listening(c);
1004 		comm_point_start_listening(c, -1, c->tcp_timeout_msec);
1005 	}
1006 }
1007 
1008 /** do the callback when reading is done */
1009 static void
1010 tcp_callback_reader(struct comm_point* c)
1011 {
1012 	log_assert(c->type == comm_tcp || c->type == comm_local);
1013 	sldns_buffer_flip(c->buffer);
1014 	if(c->tcp_do_toggle_rw)
1015 		c->tcp_is_reading = 0;
1016 	c->tcp_byte_count = 0;
1017 	if(c->tcp_req_info) {
1018 		tcp_req_info_handle_readdone(c->tcp_req_info);
1019 	} else {
1020 		if(c->type == comm_tcp)
1021 			comm_point_stop_listening(c);
1022 		fptr_ok(fptr_whitelist_comm_point(c->callback));
1023 		if( (*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &c->repinfo) ) {
1024 			comm_point_start_listening(c, -1, c->tcp_timeout_msec);
1025 		}
1026 	}
1027 }
1028 
1029 #ifdef HAVE_SSL
1030 /** log certificate details */
1031 static void
1032 log_cert(unsigned level, const char* str, X509* cert)
1033 {
1034 	BIO* bio;
1035 	char nul = 0;
1036 	char* pp = NULL;
1037 	long len;
1038 	if(verbosity < level) return;
1039 	bio = BIO_new(BIO_s_mem());
1040 	if(!bio) return;
1041 	X509_print_ex(bio, cert, 0, (unsigned long)-1
1042 		^(X509_FLAG_NO_SUBJECT
1043                         |X509_FLAG_NO_ISSUER|X509_FLAG_NO_VALIDITY
1044 			|X509_FLAG_NO_EXTENSIONS|X509_FLAG_NO_AUX
1045 			|X509_FLAG_NO_ATTRIBUTES));
1046 	BIO_write(bio, &nul, (int)sizeof(nul));
1047 	len = BIO_get_mem_data(bio, &pp);
1048 	if(len != 0 && pp) {
1049 		verbose(level, "%s: \n%s", str, pp);
1050 	}
1051 	BIO_free(bio);
1052 }
1053 #endif /* HAVE_SSL */
1054 
1055 #ifdef HAVE_SSL
1056 /** true if the ssl handshake error has to be squelched from the logs */
1057 static int
1058 squelch_err_ssl_handshake(unsigned long err)
1059 {
1060 	if(verbosity >= VERB_QUERY)
1061 		return 0; /* only squelch on low verbosity */
1062 	/* this is very specific, we could filter on ERR_GET_REASON()
1063 	 * (the third element in ERR_PACK) */
1064 	if(err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_HTTPS_PROXY_REQUEST) ||
1065 		err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_HTTP_REQUEST) ||
1066 		err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_WRONG_VERSION_NUMBER) ||
1067 		err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_READ_BYTES, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE)
1068 #ifdef SSL_F_TLS_POST_PROCESS_CLIENT_HELLO
1069 		|| err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_POST_PROCESS_CLIENT_HELLO, SSL_R_NO_SHARED_CIPHER)
1070 #endif
1071 #ifdef SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO
1072 		|| err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_UNKNOWN_PROTOCOL)
1073 		|| err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_UNSUPPORTED_PROTOCOL)
1074 #  ifdef SSL_R_VERSION_TOO_LOW
1075 		|| err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_VERSION_TOO_LOW)
1076 #  endif
1077 #endif
1078 		)
1079 		return 1;
1080 	return 0;
1081 }
1082 #endif /* HAVE_SSL */
1083 
1084 /** continue ssl handshake */
1085 #ifdef HAVE_SSL
1086 static int
1087 ssl_handshake(struct comm_point* c)
1088 {
1089 	int r;
1090 	if(c->ssl_shake_state == comm_ssl_shake_hs_read) {
1091 		/* read condition satisfied back to writing */
1092 		comm_point_listen_for_rw(c, 1, 1);
1093 		c->ssl_shake_state = comm_ssl_shake_none;
1094 		return 1;
1095 	}
1096 	if(c->ssl_shake_state == comm_ssl_shake_hs_write) {
1097 		/* write condition satisfied, back to reading */
1098 		comm_point_listen_for_rw(c, 1, 0);
1099 		c->ssl_shake_state = comm_ssl_shake_none;
1100 		return 1;
1101 	}
1102 
1103 	ERR_clear_error();
1104 	r = SSL_do_handshake(c->ssl);
1105 	if(r != 1) {
1106 		int want = SSL_get_error(c->ssl, r);
1107 		if(want == SSL_ERROR_WANT_READ) {
1108 			if(c->ssl_shake_state == comm_ssl_shake_read)
1109 				return 1;
1110 			c->ssl_shake_state = comm_ssl_shake_read;
1111 			comm_point_listen_for_rw(c, 1, 0);
1112 			return 1;
1113 		} else if(want == SSL_ERROR_WANT_WRITE) {
1114 			if(c->ssl_shake_state == comm_ssl_shake_write)
1115 				return 1;
1116 			c->ssl_shake_state = comm_ssl_shake_write;
1117 			comm_point_listen_for_rw(c, 0, 1);
1118 			return 1;
1119 		} else if(r == 0) {
1120 			return 0; /* closed */
1121 		} else if(want == SSL_ERROR_SYSCALL) {
1122 			/* SYSCALL and errno==0 means closed uncleanly */
1123 			if(errno != 0)
1124 				log_err("SSL_handshake syscall: %s",
1125 					strerror(errno));
1126 			return 0;
1127 		} else {
1128 			unsigned long err = ERR_get_error();
1129 			if(!squelch_err_ssl_handshake(err)) {
1130 				log_crypto_err_code("ssl handshake failed", err);
1131 				log_addr(VERB_OPS, "ssl handshake failed", &c->repinfo.addr,
1132 					c->repinfo.addrlen);
1133 			}
1134 			return 0;
1135 		}
1136 	}
1137 	/* this is where peer verification could take place */
1138 	if((SSL_get_verify_mode(c->ssl)&SSL_VERIFY_PEER)) {
1139 		/* verification */
1140 		if(SSL_get_verify_result(c->ssl) == X509_V_OK) {
1141 			X509* x = SSL_get_peer_certificate(c->ssl);
1142 			if(!x) {
1143 				log_addr(VERB_ALGO, "SSL connection failed: "
1144 					"no certificate",
1145 					&c->repinfo.addr, c->repinfo.addrlen);
1146 				return 0;
1147 			}
1148 			log_cert(VERB_ALGO, "peer certificate", x);
1149 #ifdef HAVE_SSL_GET0_PEERNAME
1150 			if(SSL_get0_peername(c->ssl)) {
1151 				char buf[255];
1152 				snprintf(buf, sizeof(buf), "SSL connection "
1153 					"to %s authenticated",
1154 					SSL_get0_peername(c->ssl));
1155 				log_addr(VERB_ALGO, buf, &c->repinfo.addr,
1156 					c->repinfo.addrlen);
1157 			} else {
1158 #endif
1159 				log_addr(VERB_ALGO, "SSL connection "
1160 					"authenticated", &c->repinfo.addr,
1161 					c->repinfo.addrlen);
1162 #ifdef HAVE_SSL_GET0_PEERNAME
1163 			}
1164 #endif
1165 			X509_free(x);
1166 		} else {
1167 			X509* x = SSL_get_peer_certificate(c->ssl);
1168 			if(x) {
1169 				log_cert(VERB_ALGO, "peer certificate", x);
1170 				X509_free(x);
1171 			}
1172 			log_addr(VERB_ALGO, "SSL connection failed: "
1173 				"failed to authenticate",
1174 				&c->repinfo.addr, c->repinfo.addrlen);
1175 			return 0;
1176 		}
1177 	} else {
1178 		/* unauthenticated, the verify peer flag was not set
1179 		 * in c->ssl when the ssl object was created from ssl_ctx */
1180 		log_addr(VERB_ALGO, "SSL connection", &c->repinfo.addr,
1181 			c->repinfo.addrlen);
1182 	}
1183 
1184 	/* setup listen rw correctly */
1185 	if(c->tcp_is_reading) {
1186 		if(c->ssl_shake_state != comm_ssl_shake_read)
1187 			comm_point_listen_for_rw(c, 1, 0);
1188 	} else {
1189 		comm_point_listen_for_rw(c, 1, 1);
1190 	}
1191 	c->ssl_shake_state = comm_ssl_shake_none;
1192 	return 1;
1193 }
1194 #endif /* HAVE_SSL */
1195 
1196 /** ssl read callback on TCP */
1197 static int
1198 ssl_handle_read(struct comm_point* c)
1199 {
1200 #ifdef HAVE_SSL
1201 	int r;
1202 	if(c->ssl_shake_state != comm_ssl_shake_none) {
1203 		if(!ssl_handshake(c))
1204 			return 0;
1205 		if(c->ssl_shake_state != comm_ssl_shake_none)
1206 			return 1;
1207 	}
1208 	if(c->tcp_byte_count < sizeof(uint16_t)) {
1209 		/* read length bytes */
1210 		ERR_clear_error();
1211 		if((r=SSL_read(c->ssl, (void*)sldns_buffer_at(c->buffer,
1212 			c->tcp_byte_count), (int)(sizeof(uint16_t) -
1213 			c->tcp_byte_count))) <= 0) {
1214 			int want = SSL_get_error(c->ssl, r);
1215 			if(want == SSL_ERROR_ZERO_RETURN) {
1216 				if(c->tcp_req_info)
1217 					return tcp_req_info_handle_read_close(c->tcp_req_info);
1218 				return 0; /* shutdown, closed */
1219 			} else if(want == SSL_ERROR_WANT_READ) {
1220 				ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1221 				return 1; /* read more later */
1222 			} else if(want == SSL_ERROR_WANT_WRITE) {
1223 				c->ssl_shake_state = comm_ssl_shake_hs_write;
1224 				comm_point_listen_for_rw(c, 0, 1);
1225 				return 1;
1226 			} else if(want == SSL_ERROR_SYSCALL) {
1227 #ifdef ECONNRESET
1228 				if(errno == ECONNRESET && verbosity < 2)
1229 					return 0; /* silence reset by peer */
1230 #endif
1231 				if(errno != 0)
1232 					log_err("SSL_read syscall: %s",
1233 						strerror(errno));
1234 				return 0;
1235 			}
1236 			log_crypto_err("could not SSL_read");
1237 			return 0;
1238 		}
1239 		c->tcp_byte_count += r;
1240 		if(c->tcp_byte_count < sizeof(uint16_t))
1241 			return 1;
1242 		if(sldns_buffer_read_u16_at(c->buffer, 0) >
1243 			sldns_buffer_capacity(c->buffer)) {
1244 			verbose(VERB_QUERY, "ssl: dropped larger than buffer");
1245 			return 0;
1246 		}
1247 		sldns_buffer_set_limit(c->buffer,
1248 			sldns_buffer_read_u16_at(c->buffer, 0));
1249 		if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1250 			verbose(VERB_QUERY, "ssl: dropped bogus too short.");
1251 			return 0;
1252 		}
1253 		sldns_buffer_skip(c->buffer, (ssize_t)(c->tcp_byte_count-sizeof(uint16_t)));
1254 		verbose(VERB_ALGO, "Reading ssl tcp query of length %d",
1255 			(int)sldns_buffer_limit(c->buffer));
1256 	}
1257 	if(sldns_buffer_remaining(c->buffer) > 0) {
1258 		ERR_clear_error();
1259 		r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
1260 			(int)sldns_buffer_remaining(c->buffer));
1261 		if(r <= 0) {
1262 			int want = SSL_get_error(c->ssl, r);
1263 			if(want == SSL_ERROR_ZERO_RETURN) {
1264 				if(c->tcp_req_info)
1265 					return tcp_req_info_handle_read_close(c->tcp_req_info);
1266 				return 0; /* shutdown, closed */
1267 			} else if(want == SSL_ERROR_WANT_READ) {
1268 				ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1269 				return 1; /* read more later */
1270 			} else if(want == SSL_ERROR_WANT_WRITE) {
1271 				c->ssl_shake_state = comm_ssl_shake_hs_write;
1272 				comm_point_listen_for_rw(c, 0, 1);
1273 				return 1;
1274 			} else if(want == SSL_ERROR_SYSCALL) {
1275 #ifdef ECONNRESET
1276 				if(errno == ECONNRESET && verbosity < 2)
1277 					return 0; /* silence reset by peer */
1278 #endif
1279 				if(errno != 0)
1280 					log_err("SSL_read syscall: %s",
1281 						strerror(errno));
1282 				return 0;
1283 			}
1284 			log_crypto_err("could not SSL_read");
1285 			return 0;
1286 		}
1287 		sldns_buffer_skip(c->buffer, (ssize_t)r);
1288 	}
1289 	if(sldns_buffer_remaining(c->buffer) <= 0) {
1290 		tcp_callback_reader(c);
1291 	}
1292 	return 1;
1293 #else
1294 	(void)c;
1295 	return 0;
1296 #endif /* HAVE_SSL */
1297 }
1298 
1299 /** ssl write callback on TCP */
1300 static int
1301 ssl_handle_write(struct comm_point* c)
1302 {
1303 #ifdef HAVE_SSL
1304 	int r;
1305 	if(c->ssl_shake_state != comm_ssl_shake_none) {
1306 		if(!ssl_handshake(c))
1307 			return 0;
1308 		if(c->ssl_shake_state != comm_ssl_shake_none)
1309 			return 1;
1310 	}
1311 	/* ignore return, if fails we may simply block */
1312 	(void)SSL_set_mode(c->ssl, (long)SSL_MODE_ENABLE_PARTIAL_WRITE);
1313 	if(c->tcp_byte_count < sizeof(uint16_t)) {
1314 		uint16_t len = htons(sldns_buffer_limit(c->buffer));
1315 		ERR_clear_error();
1316 		if(sizeof(uint16_t)+sldns_buffer_remaining(c->buffer) <
1317 			LDNS_RR_BUF_SIZE) {
1318 			/* combine the tcp length and the query for write,
1319 			 * this emulates writev */
1320 			uint8_t buf[LDNS_RR_BUF_SIZE];
1321 			memmove(buf, &len, sizeof(uint16_t));
1322 			memmove(buf+sizeof(uint16_t),
1323 				sldns_buffer_current(c->buffer),
1324 				sldns_buffer_remaining(c->buffer));
1325 			r = SSL_write(c->ssl, (void*)(buf+c->tcp_byte_count),
1326 				(int)(sizeof(uint16_t)+
1327 				sldns_buffer_remaining(c->buffer)
1328 				- c->tcp_byte_count));
1329 		} else {
1330 			r = SSL_write(c->ssl,
1331 				(void*)(((uint8_t*)&len)+c->tcp_byte_count),
1332 				(int)(sizeof(uint16_t)-c->tcp_byte_count));
1333 		}
1334 		if(r <= 0) {
1335 			int want = SSL_get_error(c->ssl, r);
1336 			if(want == SSL_ERROR_ZERO_RETURN) {
1337 				return 0; /* closed */
1338 			} else if(want == SSL_ERROR_WANT_READ) {
1339 				c->ssl_shake_state = comm_ssl_shake_hs_read;
1340 				comm_point_listen_for_rw(c, 1, 0);
1341 				return 1; /* wait for read condition */
1342 			} else if(want == SSL_ERROR_WANT_WRITE) {
1343 				ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1344 				return 1; /* write more later */
1345 			} else if(want == SSL_ERROR_SYSCALL) {
1346 #ifdef EPIPE
1347 				if(errno == EPIPE && verbosity < 2)
1348 					return 0; /* silence 'broken pipe' */
1349 #endif
1350 				if(errno != 0)
1351 					log_err("SSL_write syscall: %s",
1352 						strerror(errno));
1353 				return 0;
1354 			}
1355 			log_crypto_err("could not SSL_write");
1356 			return 0;
1357 		}
1358 		c->tcp_byte_count += r;
1359 		if(c->tcp_byte_count < sizeof(uint16_t))
1360 			return 1;
1361 		sldns_buffer_set_position(c->buffer, c->tcp_byte_count -
1362 			sizeof(uint16_t));
1363 		if(sldns_buffer_remaining(c->buffer) == 0) {
1364 			tcp_callback_writer(c);
1365 			return 1;
1366 		}
1367 	}
1368 	log_assert(sldns_buffer_remaining(c->buffer) > 0);
1369 	ERR_clear_error();
1370 	r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
1371 		(int)sldns_buffer_remaining(c->buffer));
1372 	if(r <= 0) {
1373 		int want = SSL_get_error(c->ssl, r);
1374 		if(want == SSL_ERROR_ZERO_RETURN) {
1375 			return 0; /* closed */
1376 		} else if(want == SSL_ERROR_WANT_READ) {
1377 			c->ssl_shake_state = comm_ssl_shake_hs_read;
1378 			comm_point_listen_for_rw(c, 1, 0);
1379 			return 1; /* wait for read condition */
1380 		} else if(want == SSL_ERROR_WANT_WRITE) {
1381 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1382 			return 1; /* write more later */
1383 		} else if(want == SSL_ERROR_SYSCALL) {
1384 #ifdef EPIPE
1385 			if(errno == EPIPE && verbosity < 2)
1386 				return 0; /* silence 'broken pipe' */
1387 #endif
1388 			if(errno != 0)
1389 				log_err("SSL_write syscall: %s",
1390 					strerror(errno));
1391 			return 0;
1392 		}
1393 		log_crypto_err("could not SSL_write");
1394 		return 0;
1395 	}
1396 	sldns_buffer_skip(c->buffer, (ssize_t)r);
1397 
1398 	if(sldns_buffer_remaining(c->buffer) == 0) {
1399 		tcp_callback_writer(c);
1400 	}
1401 	return 1;
1402 #else
1403 	(void)c;
1404 	return 0;
1405 #endif /* HAVE_SSL */
1406 }
1407 
1408 /** handle ssl tcp connection with dns contents */
1409 static int
1410 ssl_handle_it(struct comm_point* c)
1411 {
1412 	if(c->tcp_is_reading)
1413 		return ssl_handle_read(c);
1414 	return ssl_handle_write(c);
1415 }
1416 
1417 /** Handle tcp reading callback.
1418  * @param fd: file descriptor of socket.
1419  * @param c: comm point to read from into buffer.
1420  * @param short_ok: if true, very short packets are OK (for comm_local).
1421  * @return: 0 on error
1422  */
1423 static int
1424 comm_point_tcp_handle_read(int fd, struct comm_point* c, int short_ok)
1425 {
1426 	ssize_t r;
1427 	log_assert(c->type == comm_tcp || c->type == comm_local);
1428 	if(c->ssl)
1429 		return ssl_handle_it(c);
1430 	if(!c->tcp_is_reading)
1431 		return 0;
1432 
1433 	log_assert(fd != -1);
1434 	if(c->tcp_byte_count < sizeof(uint16_t)) {
1435 		/* read length bytes */
1436 		r = recv(fd,(void*)sldns_buffer_at(c->buffer,c->tcp_byte_count),
1437 			sizeof(uint16_t)-c->tcp_byte_count, 0);
1438 		if(r == 0) {
1439 			if(c->tcp_req_info)
1440 				return tcp_req_info_handle_read_close(c->tcp_req_info);
1441 			return 0;
1442 		} else if(r == -1) {
1443 #ifndef USE_WINSOCK
1444 			if(errno == EINTR || errno == EAGAIN)
1445 				return 1;
1446 #ifdef ECONNRESET
1447 			if(errno == ECONNRESET && verbosity < 2)
1448 				return 0; /* silence reset by peer */
1449 #endif
1450 			log_err_addr("read (in tcp s)", strerror(errno),
1451 				&c->repinfo.addr, c->repinfo.addrlen);
1452 #else /* USE_WINSOCK */
1453 			if(WSAGetLastError() == WSAECONNRESET)
1454 				return 0;
1455 			if(WSAGetLastError() == WSAEINPROGRESS)
1456 				return 1;
1457 			if(WSAGetLastError() == WSAEWOULDBLOCK) {
1458 				ub_winsock_tcp_wouldblock(c->ev->ev,
1459 					UB_EV_READ);
1460 				return 1;
1461 			}
1462 			log_err_addr("read (in tcp s)",
1463 				wsa_strerror(WSAGetLastError()),
1464 				&c->repinfo.addr, c->repinfo.addrlen);
1465 #endif
1466 			return 0;
1467 		}
1468 		c->tcp_byte_count += r;
1469 		if(c->tcp_byte_count != sizeof(uint16_t))
1470 			return 1;
1471 		if(sldns_buffer_read_u16_at(c->buffer, 0) >
1472 			sldns_buffer_capacity(c->buffer)) {
1473 			verbose(VERB_QUERY, "tcp: dropped larger than buffer");
1474 			return 0;
1475 		}
1476 		sldns_buffer_set_limit(c->buffer,
1477 			sldns_buffer_read_u16_at(c->buffer, 0));
1478 		if(!short_ok &&
1479 			sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1480 			verbose(VERB_QUERY, "tcp: dropped bogus too short.");
1481 			return 0;
1482 		}
1483 		verbose(VERB_ALGO, "Reading tcp query of length %d",
1484 			(int)sldns_buffer_limit(c->buffer));
1485 	}
1486 
1487 	log_assert(sldns_buffer_remaining(c->buffer) > 0);
1488 	r = recv(fd, (void*)sldns_buffer_current(c->buffer),
1489 		sldns_buffer_remaining(c->buffer), 0);
1490 	if(r == 0) {
1491 		if(c->tcp_req_info)
1492 			return tcp_req_info_handle_read_close(c->tcp_req_info);
1493 		return 0;
1494 	} else if(r == -1) {
1495 #ifndef USE_WINSOCK
1496 		if(errno == EINTR || errno == EAGAIN)
1497 			return 1;
1498 		log_err_addr("read (in tcp r)", strerror(errno),
1499 			&c->repinfo.addr, c->repinfo.addrlen);
1500 #else /* USE_WINSOCK */
1501 		if(WSAGetLastError() == WSAECONNRESET)
1502 			return 0;
1503 		if(WSAGetLastError() == WSAEINPROGRESS)
1504 			return 1;
1505 		if(WSAGetLastError() == WSAEWOULDBLOCK) {
1506 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1507 			return 1;
1508 		}
1509 		log_err_addr("read (in tcp r)",
1510 			wsa_strerror(WSAGetLastError()),
1511 			&c->repinfo.addr, c->repinfo.addrlen);
1512 #endif
1513 		return 0;
1514 	}
1515 	sldns_buffer_skip(c->buffer, r);
1516 	if(sldns_buffer_remaining(c->buffer) <= 0) {
1517 		tcp_callback_reader(c);
1518 	}
1519 	return 1;
1520 }
1521 
1522 /**
1523  * Handle tcp writing callback.
1524  * @param fd: file descriptor of socket.
1525  * @param c: comm point to write buffer out of.
1526  * @return: 0 on error
1527  */
1528 static int
1529 comm_point_tcp_handle_write(int fd, struct comm_point* c)
1530 {
1531 	ssize_t r;
1532 	struct sldns_buffer *buffer;
1533 	log_assert(c->type == comm_tcp);
1534 #ifdef USE_DNSCRYPT
1535 	buffer = c->dnscrypt_buffer;
1536 #else
1537 	buffer = c->buffer;
1538 #endif
1539 	if(c->tcp_is_reading && !c->ssl)
1540 		return 0;
1541 	log_assert(fd != -1);
1542 	if(c->tcp_byte_count == 0 && c->tcp_check_nb_connect) {
1543 		/* check for pending error from nonblocking connect */
1544 		/* from Stevens, unix network programming, vol1, 3rd ed, p450*/
1545 		int error = 0;
1546 		socklen_t len = (socklen_t)sizeof(error);
1547 		if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
1548 			&len) < 0){
1549 #ifndef USE_WINSOCK
1550 			error = errno; /* on solaris errno is error */
1551 #else /* USE_WINSOCK */
1552 			error = WSAGetLastError();
1553 #endif
1554 		}
1555 #ifndef USE_WINSOCK
1556 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
1557 		if(error == EINPROGRESS || error == EWOULDBLOCK)
1558 			return 1; /* try again later */
1559 		else
1560 #endif
1561 		if(error != 0 && verbosity < 2)
1562 			return 0; /* silence lots of chatter in the logs */
1563                 else if(error != 0) {
1564 			log_err_addr("tcp connect", strerror(error),
1565 				&c->repinfo.addr, c->repinfo.addrlen);
1566 #else /* USE_WINSOCK */
1567 		/* examine error */
1568 		if(error == WSAEINPROGRESS)
1569 			return 1;
1570 		else if(error == WSAEWOULDBLOCK) {
1571 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1572 			return 1;
1573 		} else if(error != 0 && verbosity < 2)
1574 			return 0;
1575 		else if(error != 0) {
1576 			log_err_addr("tcp connect", wsa_strerror(error),
1577 				&c->repinfo.addr, c->repinfo.addrlen);
1578 #endif /* USE_WINSOCK */
1579 			return 0;
1580 		}
1581 	}
1582 	if(c->ssl)
1583 		return ssl_handle_it(c);
1584 
1585 #ifdef USE_MSG_FASTOPEN
1586 	/* Only try this on first use of a connection that uses tfo,
1587 	   otherwise fall through to normal write */
1588 	/* Also, TFO support on WINDOWS not implemented at the moment */
1589 	if(c->tcp_do_fastopen == 1) {
1590 		/* this form of sendmsg() does both a connect() and send() so need to
1591 		   look for various flavours of error*/
1592 		uint16_t len = htons(sldns_buffer_limit(buffer));
1593 		struct msghdr msg;
1594 		struct iovec iov[2];
1595 		c->tcp_do_fastopen = 0;
1596 		memset(&msg, 0, sizeof(msg));
1597 		iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
1598 		iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
1599 		iov[1].iov_base = sldns_buffer_begin(buffer);
1600 		iov[1].iov_len = sldns_buffer_limit(buffer);
1601 		log_assert(iov[0].iov_len > 0);
1602 		msg.msg_name = &c->repinfo.addr;
1603 		msg.msg_namelen = c->repinfo.addrlen;
1604 		msg.msg_iov = iov;
1605 		msg.msg_iovlen = 2;
1606 		r = sendmsg(fd, &msg, MSG_FASTOPEN);
1607 		if (r == -1) {
1608 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
1609 			/* Handshake is underway, maybe because no TFO cookie available.
1610 			   Come back to write the message*/
1611 			if(errno == EINPROGRESS || errno == EWOULDBLOCK)
1612 				return 1;
1613 #endif
1614 			if(errno == EINTR || errno == EAGAIN)
1615 				return 1;
1616 			/* Not handling EISCONN here as shouldn't ever hit that case.*/
1617 			if(errno != EPIPE && errno != 0 && verbosity < 2)
1618 				return 0; /* silence lots of chatter in the logs */
1619 			if(errno != EPIPE && errno != 0) {
1620 				log_err_addr("tcp sendmsg", strerror(errno),
1621 					&c->repinfo.addr, c->repinfo.addrlen);
1622 				return 0;
1623 			}
1624 			/* fallthrough to nonFASTOPEN
1625 			 * (MSG_FASTOPEN on Linux 3 produces EPIPE)
1626 			 * we need to perform connect() */
1627 			if(connect(fd, (struct sockaddr *)&c->repinfo.addr, c->repinfo.addrlen) == -1) {
1628 #ifdef EINPROGRESS
1629 				if(errno == EINPROGRESS)
1630 					return 1; /* wait until connect done*/
1631 #endif
1632 #ifdef USE_WINSOCK
1633 				if(WSAGetLastError() == WSAEINPROGRESS ||
1634 					WSAGetLastError() == WSAEWOULDBLOCK)
1635 					return 1; /* wait until connect done*/
1636 #endif
1637 				if(tcp_connect_errno_needs_log(
1638 					(struct sockaddr *)&c->repinfo.addr, c->repinfo.addrlen)) {
1639 					log_err_addr("outgoing tcp: connect after EPIPE for fastopen",
1640 						strerror(errno), &c->repinfo.addr, c->repinfo.addrlen);
1641 				}
1642 				return 0;
1643 			}
1644 
1645 		} else {
1646 			c->tcp_byte_count += r;
1647 			if(c->tcp_byte_count < sizeof(uint16_t))
1648 				return 1;
1649 			sldns_buffer_set_position(buffer, c->tcp_byte_count -
1650 				sizeof(uint16_t));
1651 			if(sldns_buffer_remaining(buffer) == 0) {
1652 				tcp_callback_writer(c);
1653 				return 1;
1654 			}
1655 		}
1656 	}
1657 #endif /* USE_MSG_FASTOPEN */
1658 
1659 	if(c->tcp_byte_count < sizeof(uint16_t)) {
1660 		uint16_t len = htons(sldns_buffer_limit(buffer));
1661 #ifdef HAVE_WRITEV
1662 		struct iovec iov[2];
1663 		iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
1664 		iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
1665 		iov[1].iov_base = sldns_buffer_begin(buffer);
1666 		iov[1].iov_len = sldns_buffer_limit(buffer);
1667 		log_assert(iov[0].iov_len > 0);
1668 		r = writev(fd, iov, 2);
1669 #else /* HAVE_WRITEV */
1670 		r = send(fd, (void*)(((uint8_t*)&len)+c->tcp_byte_count),
1671 			sizeof(uint16_t)-c->tcp_byte_count, 0);
1672 #endif /* HAVE_WRITEV */
1673 		if(r == -1) {
1674 #ifndef USE_WINSOCK
1675 #  ifdef EPIPE
1676                 	if(errno == EPIPE && verbosity < 2)
1677                         	return 0; /* silence 'broken pipe' */
1678   #endif
1679 			if(errno == EINTR || errno == EAGAIN)
1680 				return 1;
1681 #ifdef ECONNRESET
1682 			if(errno == ECONNRESET && verbosity < 2)
1683 				return 0; /* silence reset by peer */
1684 #endif
1685 #  ifdef HAVE_WRITEV
1686 			log_err_addr("tcp writev", strerror(errno),
1687 				&c->repinfo.addr, c->repinfo.addrlen);
1688 #  else /* HAVE_WRITEV */
1689 			log_err_addr("tcp send s", strerror(errno),
1690 				&c->repinfo.addr, c->repinfo.addrlen);
1691 #  endif /* HAVE_WRITEV */
1692 #else
1693 			if(WSAGetLastError() == WSAENOTCONN)
1694 				return 1;
1695 			if(WSAGetLastError() == WSAEINPROGRESS)
1696 				return 1;
1697 			if(WSAGetLastError() == WSAEWOULDBLOCK) {
1698 				ub_winsock_tcp_wouldblock(c->ev->ev,
1699 					UB_EV_WRITE);
1700 				return 1;
1701 			}
1702 			if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
1703 				return 0; /* silence reset by peer */
1704 			log_err_addr("tcp send s",
1705 				wsa_strerror(WSAGetLastError()),
1706 				&c->repinfo.addr, c->repinfo.addrlen);
1707 #endif
1708 			return 0;
1709 		}
1710 		c->tcp_byte_count += r;
1711 		if(c->tcp_byte_count < sizeof(uint16_t))
1712 			return 1;
1713 		sldns_buffer_set_position(buffer, c->tcp_byte_count -
1714 			sizeof(uint16_t));
1715 		if(sldns_buffer_remaining(buffer) == 0) {
1716 			tcp_callback_writer(c);
1717 			return 1;
1718 		}
1719 	}
1720 	log_assert(sldns_buffer_remaining(buffer) > 0);
1721 	r = send(fd, (void*)sldns_buffer_current(buffer),
1722 		sldns_buffer_remaining(buffer), 0);
1723 	if(r == -1) {
1724 #ifndef USE_WINSOCK
1725 		if(errno == EINTR || errno == EAGAIN)
1726 			return 1;
1727 #ifdef ECONNRESET
1728 		if(errno == ECONNRESET && verbosity < 2)
1729 			return 0; /* silence reset by peer */
1730 #endif
1731 		log_err_addr("tcp send r", strerror(errno),
1732 			&c->repinfo.addr, c->repinfo.addrlen);
1733 #else
1734 		if(WSAGetLastError() == WSAEINPROGRESS)
1735 			return 1;
1736 		if(WSAGetLastError() == WSAEWOULDBLOCK) {
1737 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1738 			return 1;
1739 		}
1740 		if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
1741 			return 0; /* silence reset by peer */
1742 		log_err_addr("tcp send r", wsa_strerror(WSAGetLastError()),
1743 			&c->repinfo.addr, c->repinfo.addrlen);
1744 #endif
1745 		return 0;
1746 	}
1747 	sldns_buffer_skip(buffer, r);
1748 
1749 	if(sldns_buffer_remaining(buffer) == 0) {
1750 		tcp_callback_writer(c);
1751 	}
1752 
1753 	return 1;
1754 }
1755 
1756 /** read again to drain buffers when there could be more to read */
1757 static void
1758 tcp_req_info_read_again(int fd, struct comm_point* c)
1759 {
1760 	while(c->tcp_req_info->read_again) {
1761 		int r;
1762 		c->tcp_req_info->read_again = 0;
1763 		if(c->tcp_is_reading)
1764 			r = comm_point_tcp_handle_read(fd, c, 0);
1765 		else 	r = comm_point_tcp_handle_write(fd, c);
1766 		if(!r) {
1767 			reclaim_tcp_handler(c);
1768 			if(!c->tcp_do_close) {
1769 				fptr_ok(fptr_whitelist_comm_point(
1770 					c->callback));
1771 				(void)(*c->callback)(c, c->cb_arg,
1772 					NETEVENT_CLOSED, NULL);
1773 			}
1774 			return;
1775 		}
1776 	}
1777 }
1778 
1779 void
1780 comm_point_tcp_handle_callback(int fd, short event, void* arg)
1781 {
1782 	struct comm_point* c = (struct comm_point*)arg;
1783 	log_assert(c->type == comm_tcp);
1784 	ub_comm_base_now(c->ev->base);
1785 
1786 #ifdef USE_DNSCRYPT
1787 	/* Initialize if this is a dnscrypt socket */
1788 	if(c->tcp_parent) {
1789 		c->dnscrypt = c->tcp_parent->dnscrypt;
1790 	}
1791 	if(c->dnscrypt && c->dnscrypt_buffer == c->buffer) {
1792 		c->dnscrypt_buffer = sldns_buffer_new(sldns_buffer_capacity(c->buffer));
1793 		if(!c->dnscrypt_buffer) {
1794 			log_err("Could not allocate dnscrypt buffer");
1795 			reclaim_tcp_handler(c);
1796 			if(!c->tcp_do_close) {
1797 				fptr_ok(fptr_whitelist_comm_point(
1798 					c->callback));
1799 				(void)(*c->callback)(c, c->cb_arg,
1800 					NETEVENT_CLOSED, NULL);
1801 			}
1802 			return;
1803 		}
1804 	}
1805 #endif
1806 
1807 	if(event&UB_EV_TIMEOUT) {
1808 		verbose(VERB_QUERY, "tcp took too long, dropped");
1809 		reclaim_tcp_handler(c);
1810 		if(!c->tcp_do_close) {
1811 			fptr_ok(fptr_whitelist_comm_point(c->callback));
1812 			(void)(*c->callback)(c, c->cb_arg,
1813 				NETEVENT_TIMEOUT, NULL);
1814 		}
1815 		return;
1816 	}
1817 	if(event&UB_EV_READ) {
1818 		int has_tcpq = (c->tcp_req_info != NULL);
1819 		if(!comm_point_tcp_handle_read(fd, c, 0)) {
1820 			reclaim_tcp_handler(c);
1821 			if(!c->tcp_do_close) {
1822 				fptr_ok(fptr_whitelist_comm_point(
1823 					c->callback));
1824 				(void)(*c->callback)(c, c->cb_arg,
1825 					NETEVENT_CLOSED, NULL);
1826 			}
1827 		}
1828 		if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again)
1829 			tcp_req_info_read_again(fd, c);
1830 		return;
1831 	}
1832 	if(event&UB_EV_WRITE) {
1833 		int has_tcpq = (c->tcp_req_info != NULL);
1834 		if(!comm_point_tcp_handle_write(fd, c)) {
1835 			reclaim_tcp_handler(c);
1836 			if(!c->tcp_do_close) {
1837 				fptr_ok(fptr_whitelist_comm_point(
1838 					c->callback));
1839 				(void)(*c->callback)(c, c->cb_arg,
1840 					NETEVENT_CLOSED, NULL);
1841 			}
1842 		}
1843 		if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again)
1844 			tcp_req_info_read_again(fd, c);
1845 		return;
1846 	}
1847 	log_err("Ignored event %d for tcphdl.", event);
1848 }
1849 
1850 /** Make http handler free for next assignment */
1851 static void
1852 reclaim_http_handler(struct comm_point* c)
1853 {
1854 	log_assert(c->type == comm_http);
1855 	if(c->ssl) {
1856 #ifdef HAVE_SSL
1857 		SSL_shutdown(c->ssl);
1858 		SSL_free(c->ssl);
1859 		c->ssl = NULL;
1860 #endif
1861 	}
1862 	comm_point_close(c);
1863 	if(c->tcp_parent) {
1864 		c->tcp_parent->cur_tcp_count--;
1865 		c->tcp_free = c->tcp_parent->tcp_free;
1866 		c->tcp_parent->tcp_free = c;
1867 		if(!c->tcp_free) {
1868 			/* re-enable listening on accept socket */
1869 			comm_point_start_listening(c->tcp_parent, -1, -1);
1870 		}
1871 	}
1872 }
1873 
1874 /** read more data for http (with ssl) */
1875 static int
1876 ssl_http_read_more(struct comm_point* c)
1877 {
1878 #ifdef HAVE_SSL
1879 	int r;
1880 	log_assert(sldns_buffer_remaining(c->buffer) > 0);
1881 	ERR_clear_error();
1882 	r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
1883 		(int)sldns_buffer_remaining(c->buffer));
1884 	if(r <= 0) {
1885 		int want = SSL_get_error(c->ssl, r);
1886 		if(want == SSL_ERROR_ZERO_RETURN) {
1887 			return 0; /* shutdown, closed */
1888 		} else if(want == SSL_ERROR_WANT_READ) {
1889 			return 1; /* read more later */
1890 		} else if(want == SSL_ERROR_WANT_WRITE) {
1891 			c->ssl_shake_state = comm_ssl_shake_hs_write;
1892 			comm_point_listen_for_rw(c, 0, 1);
1893 			return 1;
1894 		} else if(want == SSL_ERROR_SYSCALL) {
1895 #ifdef ECONNRESET
1896 			if(errno == ECONNRESET && verbosity < 2)
1897 				return 0; /* silence reset by peer */
1898 #endif
1899 			if(errno != 0)
1900 				log_err("SSL_read syscall: %s",
1901 					strerror(errno));
1902 			return 0;
1903 		}
1904 		log_crypto_err("could not SSL_read");
1905 		return 0;
1906 	}
1907 	sldns_buffer_skip(c->buffer, (ssize_t)r);
1908 	return 1;
1909 #else
1910 	(void)c;
1911 	return 0;
1912 #endif /* HAVE_SSL */
1913 }
1914 
1915 /** read more data for http */
1916 static int
1917 http_read_more(int fd, struct comm_point* c)
1918 {
1919 	ssize_t r;
1920 	log_assert(sldns_buffer_remaining(c->buffer) > 0);
1921 	r = recv(fd, (void*)sldns_buffer_current(c->buffer),
1922 		sldns_buffer_remaining(c->buffer), 0);
1923 	if(r == 0) {
1924 		return 0;
1925 	} else if(r == -1) {
1926 #ifndef USE_WINSOCK
1927 		if(errno == EINTR || errno == EAGAIN)
1928 			return 1;
1929 		log_err_addr("read (in http r)", strerror(errno),
1930 			&c->repinfo.addr, c->repinfo.addrlen);
1931 #else /* USE_WINSOCK */
1932 		if(WSAGetLastError() == WSAECONNRESET)
1933 			return 0;
1934 		if(WSAGetLastError() == WSAEINPROGRESS)
1935 			return 1;
1936 		if(WSAGetLastError() == WSAEWOULDBLOCK) {
1937 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1938 			return 1;
1939 		}
1940 		log_err_addr("read (in http r)",
1941 			wsa_strerror(WSAGetLastError()),
1942 			&c->repinfo.addr, c->repinfo.addrlen);
1943 #endif
1944 		return 0;
1945 	}
1946 	sldns_buffer_skip(c->buffer, r);
1947 	return 1;
1948 }
1949 
1950 /** return true if http header has been read (one line complete) */
1951 static int
1952 http_header_done(sldns_buffer* buf)
1953 {
1954 	size_t i;
1955 	for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
1956 		/* there was a \r before the \n, but we ignore that */
1957 		if((char)sldns_buffer_read_u8_at(buf, i) == '\n')
1958 			return 1;
1959 	}
1960 	return 0;
1961 }
1962 
1963 /** return character string into buffer for header line, moves buffer
1964  * past that line and puts zero terminator into linefeed-newline */
1965 static char*
1966 http_header_line(sldns_buffer* buf)
1967 {
1968 	char* result = (char*)sldns_buffer_current(buf);
1969 	size_t i;
1970 	for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
1971 		/* terminate the string on the \r */
1972 		if((char)sldns_buffer_read_u8_at(buf, i) == '\r')
1973 			sldns_buffer_write_u8_at(buf, i, 0);
1974 		/* terminate on the \n and skip past the it and done */
1975 		if((char)sldns_buffer_read_u8_at(buf, i) == '\n') {
1976 			sldns_buffer_write_u8_at(buf, i, 0);
1977 			sldns_buffer_set_position(buf, i+1);
1978 			return result;
1979 		}
1980 	}
1981 	return NULL;
1982 }
1983 
1984 /** move unread buffer to start and clear rest for putting the rest into it */
1985 static void
1986 http_moveover_buffer(sldns_buffer* buf)
1987 {
1988 	size_t pos = sldns_buffer_position(buf);
1989 	size_t len = sldns_buffer_remaining(buf);
1990 	sldns_buffer_clear(buf);
1991 	memmove(sldns_buffer_begin(buf), sldns_buffer_at(buf, pos), len);
1992 	sldns_buffer_set_position(buf, len);
1993 }
1994 
1995 /** a http header is complete, process it */
1996 static int
1997 http_process_initial_header(struct comm_point* c)
1998 {
1999 	char* line = http_header_line(c->buffer);
2000 	if(!line) return 1;
2001 	verbose(VERB_ALGO, "http header: %s", line);
2002 	if(strncasecmp(line, "HTTP/1.1 ", 9) == 0) {
2003 		/* check returncode */
2004 		if(line[9] != '2') {
2005 			verbose(VERB_ALGO, "http bad status %s", line+9);
2006 			return 0;
2007 		}
2008 	} else if(strncasecmp(line, "Content-Length: ", 16) == 0) {
2009 		if(!c->http_is_chunked)
2010 			c->tcp_byte_count = (size_t)atoi(line+16);
2011 	} else if(strncasecmp(line, "Transfer-Encoding: chunked", 19+7) == 0) {
2012 		c->tcp_byte_count = 0;
2013 		c->http_is_chunked = 1;
2014 	} else if(line[0] == 0) {
2015 		/* end of initial headers */
2016 		c->http_in_headers = 0;
2017 		if(c->http_is_chunked)
2018 			c->http_in_chunk_headers = 1;
2019 		/* remove header text from front of buffer
2020 		 * the buffer is going to be used to return the data segment
2021 		 * itself and we don't want the header to get returned
2022 		 * prepended with it */
2023 		http_moveover_buffer(c->buffer);
2024 		sldns_buffer_flip(c->buffer);
2025 		return 1;
2026 	}
2027 	/* ignore other headers */
2028 	return 1;
2029 }
2030 
2031 /** a chunk header is complete, process it, return 0=fail, 1=continue next
2032  * header line, 2=done with chunked transfer*/
2033 static int
2034 http_process_chunk_header(struct comm_point* c)
2035 {
2036 	char* line = http_header_line(c->buffer);
2037 	if(!line) return 1;
2038 	if(c->http_in_chunk_headers == 3) {
2039 		verbose(VERB_ALGO, "http chunk trailer: %s", line);
2040 		/* are we done ? */
2041 		if(line[0] == 0 && c->tcp_byte_count == 0) {
2042 			/* callback of http reader when NETEVENT_DONE,
2043 			 * end of data, with no data in buffer */
2044 			sldns_buffer_set_position(c->buffer, 0);
2045 			sldns_buffer_set_limit(c->buffer, 0);
2046 			fptr_ok(fptr_whitelist_comm_point(c->callback));
2047 			(void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
2048 			/* return that we are done */
2049 			return 2;
2050 		}
2051 		if(line[0] == 0) {
2052 			/* continue with header of the next chunk */
2053 			c->http_in_chunk_headers = 1;
2054 			/* remove header text from front of buffer */
2055 			http_moveover_buffer(c->buffer);
2056 			sldns_buffer_flip(c->buffer);
2057 			return 1;
2058 		}
2059 		/* ignore further trail headers */
2060 		return 1;
2061 	}
2062 	verbose(VERB_ALGO, "http chunk header: %s", line);
2063 	if(c->http_in_chunk_headers == 1) {
2064 		/* read chunked start line */
2065 		char* end = NULL;
2066 		c->tcp_byte_count = (size_t)strtol(line, &end, 16);
2067 		if(end == line)
2068 			return 0;
2069 		c->http_in_chunk_headers = 0;
2070 		/* remove header text from front of buffer */
2071 		http_moveover_buffer(c->buffer);
2072 		sldns_buffer_flip(c->buffer);
2073 		if(c->tcp_byte_count == 0) {
2074 			/* done with chunks, process chunk_trailer lines */
2075 			c->http_in_chunk_headers = 3;
2076 		}
2077 		return 1;
2078 	}
2079 	/* ignore other headers */
2080 	return 1;
2081 }
2082 
2083 /** handle nonchunked data segment */
2084 static int
2085 http_nonchunk_segment(struct comm_point* c)
2086 {
2087 	/* c->buffer at position..limit has new data we read in.
2088 	 * the buffer itself is full of nonchunked data.
2089 	 * we are looking to read tcp_byte_count more data
2090 	 * and then the transfer is done. */
2091 	size_t remainbufferlen;
2092 	size_t got_now = sldns_buffer_limit(c->buffer) - c->http_stored;
2093 	if(c->tcp_byte_count <= got_now) {
2094 		/* done, this is the last data fragment */
2095 		c->http_stored = 0;
2096 		sldns_buffer_set_position(c->buffer, 0);
2097 		fptr_ok(fptr_whitelist_comm_point(c->callback));
2098 		(void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
2099 		return 1;
2100 	}
2101 	c->tcp_byte_count -= got_now;
2102 	/* if we have the buffer space,
2103 	 * read more data collected into the buffer */
2104 	remainbufferlen = sldns_buffer_capacity(c->buffer) -
2105 		sldns_buffer_limit(c->buffer);
2106 	if(remainbufferlen >= c->tcp_byte_count ||
2107 		remainbufferlen >= 2048) {
2108 		size_t total = sldns_buffer_limit(c->buffer);
2109 		sldns_buffer_clear(c->buffer);
2110 		sldns_buffer_set_position(c->buffer, total);
2111 		c->http_stored = total;
2112 		/* return and wait to read more */
2113 		return 1;
2114 	}
2115 	/* call callback with this data amount, then
2116 	 * wait for more */
2117 	c->http_stored = 0;
2118 	sldns_buffer_set_position(c->buffer, 0);
2119 	fptr_ok(fptr_whitelist_comm_point(c->callback));
2120 	(void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
2121 	/* c->callback has to buffer_clear(c->buffer). */
2122 	/* return and wait to read more */
2123 	return 1;
2124 }
2125 
2126 /** handle nonchunked data segment, return 0=fail, 1=wait, 2=process more */
2127 static int
2128 http_chunked_segment(struct comm_point* c)
2129 {
2130 	/* the c->buffer has from position..limit new data we read. */
2131 	/* the current chunk has length tcp_byte_count.
2132 	 * once we read that read more chunk headers.
2133 	 */
2134 	size_t remainbufferlen;
2135 	size_t got_now = sldns_buffer_limit(c->buffer) - c->http_stored;
2136 	if(c->tcp_byte_count <= got_now) {
2137 		/* the chunk has completed (with perhaps some extra data
2138 		 * from next chunk header and next chunk) */
2139 		/* save too much info into temp buffer */
2140 		size_t fraglen;
2141 		struct comm_reply repinfo;
2142 		c->http_stored = 0;
2143 		sldns_buffer_skip(c->buffer, (ssize_t)c->tcp_byte_count);
2144 		sldns_buffer_clear(c->http_temp);
2145 		sldns_buffer_write(c->http_temp,
2146 			sldns_buffer_current(c->buffer),
2147 			sldns_buffer_remaining(c->buffer));
2148 		sldns_buffer_flip(c->http_temp);
2149 
2150 		/* callback with this fragment */
2151 		fraglen = sldns_buffer_position(c->buffer);
2152 		sldns_buffer_set_position(c->buffer, 0);
2153 		sldns_buffer_set_limit(c->buffer, fraglen);
2154 		repinfo = c->repinfo;
2155 		fptr_ok(fptr_whitelist_comm_point(c->callback));
2156 		(void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &repinfo);
2157 		/* c->callback has to buffer_clear(). */
2158 
2159 		/* is commpoint deleted? */
2160 		if(!repinfo.c) {
2161 			return 1;
2162 		}
2163 		/* copy waiting info */
2164 		sldns_buffer_clear(c->buffer);
2165 		sldns_buffer_write(c->buffer,
2166 			sldns_buffer_begin(c->http_temp),
2167 			sldns_buffer_remaining(c->http_temp));
2168 		sldns_buffer_flip(c->buffer);
2169 		/* process end of chunk trailer header lines, until
2170 		 * an empty line */
2171 		c->http_in_chunk_headers = 3;
2172 		/* process more data in buffer (if any) */
2173 		return 2;
2174 	}
2175 	c->tcp_byte_count -= got_now;
2176 
2177 	/* if we have the buffer space,
2178 	 * read more data collected into the buffer */
2179 	remainbufferlen = sldns_buffer_capacity(c->buffer) -
2180 		sldns_buffer_limit(c->buffer);
2181 	if(remainbufferlen >= c->tcp_byte_count ||
2182 		remainbufferlen >= 2048) {
2183 		size_t total = sldns_buffer_limit(c->buffer);
2184 		sldns_buffer_clear(c->buffer);
2185 		sldns_buffer_set_position(c->buffer, total);
2186 		c->http_stored = total;
2187 		/* return and wait to read more */
2188 		return 1;
2189 	}
2190 
2191 	/* callback of http reader for a new part of the data */
2192 	c->http_stored = 0;
2193 	sldns_buffer_set_position(c->buffer, 0);
2194 	fptr_ok(fptr_whitelist_comm_point(c->callback));
2195 	(void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
2196 	/* c->callback has to buffer_clear(c->buffer). */
2197 	/* return and wait to read more */
2198 	return 1;
2199 }
2200 
2201 /**
2202  * Handle http reading callback.
2203  * @param fd: file descriptor of socket.
2204  * @param c: comm point to read from into buffer.
2205  * @return: 0 on error
2206  */
2207 static int
2208 comm_point_http_handle_read(int fd, struct comm_point* c)
2209 {
2210 	log_assert(c->type == comm_http);
2211 	log_assert(fd != -1);
2212 
2213 	/* if we are in ssl handshake, handle SSL handshake */
2214 #ifdef HAVE_SSL
2215 	if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
2216 		if(!ssl_handshake(c))
2217 			return 0;
2218 		if(c->ssl_shake_state != comm_ssl_shake_none)
2219 			return 1;
2220 	}
2221 #endif /* HAVE_SSL */
2222 
2223 	if(!c->tcp_is_reading)
2224 		return 1;
2225 	/* read more data */
2226 	if(c->ssl) {
2227 		if(!ssl_http_read_more(c))
2228 			return 0;
2229 	} else {
2230 		if(!http_read_more(fd, c))
2231 			return 0;
2232 	}
2233 
2234 	sldns_buffer_flip(c->buffer);
2235 	while(sldns_buffer_remaining(c->buffer) > 0) {
2236 		/* if we are reading headers, read more headers */
2237 		if(c->http_in_headers || c->http_in_chunk_headers) {
2238 			/* if header is done, process the header */
2239 			if(!http_header_done(c->buffer)) {
2240 				/* copy remaining data to front of buffer
2241 				 * and set rest for writing into it */
2242 				http_moveover_buffer(c->buffer);
2243 				/* return and wait to read more */
2244 				return 1;
2245 			}
2246 			if(!c->http_in_chunk_headers) {
2247 				/* process initial headers */
2248 				if(!http_process_initial_header(c))
2249 					return 0;
2250 			} else {
2251 				/* process chunk headers */
2252 				int r = http_process_chunk_header(c);
2253 				if(r == 0) return 0;
2254 				if(r == 2) return 1; /* done */
2255 				/* r == 1, continue */
2256 			}
2257 			/* see if we have more to process */
2258 			continue;
2259 		}
2260 
2261 		if(!c->http_is_chunked) {
2262 			/* if we are reading nonchunks, process that*/
2263 			return http_nonchunk_segment(c);
2264 		} else {
2265 			/* if we are reading chunks, read the chunk */
2266 			int r = http_chunked_segment(c);
2267 			if(r == 0) return 0;
2268 			if(r == 1) return 1;
2269 			continue;
2270 		}
2271 	}
2272 	/* broke out of the loop; could not process header instead need
2273 	 * to read more */
2274 	/* moveover any remaining data and read more data */
2275 	http_moveover_buffer(c->buffer);
2276 	/* return and wait to read more */
2277 	return 1;
2278 }
2279 
2280 /** check pending connect for http */
2281 static int
2282 http_check_connect(int fd, struct comm_point* c)
2283 {
2284 	/* check for pending error from nonblocking connect */
2285 	/* from Stevens, unix network programming, vol1, 3rd ed, p450*/
2286 	int error = 0;
2287 	socklen_t len = (socklen_t)sizeof(error);
2288 	if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
2289 		&len) < 0){
2290 #ifndef USE_WINSOCK
2291 		error = errno; /* on solaris errno is error */
2292 #else /* USE_WINSOCK */
2293 		error = WSAGetLastError();
2294 #endif
2295 	}
2296 #ifndef USE_WINSOCK
2297 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
2298 	if(error == EINPROGRESS || error == EWOULDBLOCK)
2299 		return 1; /* try again later */
2300 	else
2301 #endif
2302 	if(error != 0 && verbosity < 2)
2303 		return 0; /* silence lots of chatter in the logs */
2304 	else if(error != 0) {
2305 		log_err_addr("http connect", strerror(error),
2306 			&c->repinfo.addr, c->repinfo.addrlen);
2307 #else /* USE_WINSOCK */
2308 	/* examine error */
2309 	if(error == WSAEINPROGRESS)
2310 		return 1;
2311 	else if(error == WSAEWOULDBLOCK) {
2312 		ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2313 		return 1;
2314 	} else if(error != 0 && verbosity < 2)
2315 		return 0;
2316 	else if(error != 0) {
2317 		log_err_addr("http connect", wsa_strerror(error),
2318 			&c->repinfo.addr, c->repinfo.addrlen);
2319 #endif /* USE_WINSOCK */
2320 		return 0;
2321 	}
2322 	/* keep on processing this socket */
2323 	return 2;
2324 }
2325 
2326 /** write more data for http (with ssl) */
2327 static int
2328 ssl_http_write_more(struct comm_point* c)
2329 {
2330 #ifdef HAVE_SSL
2331 	int r;
2332 	log_assert(sldns_buffer_remaining(c->buffer) > 0);
2333 	ERR_clear_error();
2334 	r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
2335 		(int)sldns_buffer_remaining(c->buffer));
2336 	if(r <= 0) {
2337 		int want = SSL_get_error(c->ssl, r);
2338 		if(want == SSL_ERROR_ZERO_RETURN) {
2339 			return 0; /* closed */
2340 		} else if(want == SSL_ERROR_WANT_READ) {
2341 			c->ssl_shake_state = comm_ssl_shake_hs_read;
2342 			comm_point_listen_for_rw(c, 1, 0);
2343 			return 1; /* wait for read condition */
2344 		} else if(want == SSL_ERROR_WANT_WRITE) {
2345 			return 1; /* write more later */
2346 		} else if(want == SSL_ERROR_SYSCALL) {
2347 #ifdef EPIPE
2348 			if(errno == EPIPE && verbosity < 2)
2349 				return 0; /* silence 'broken pipe' */
2350 #endif
2351 			if(errno != 0)
2352 				log_err("SSL_write syscall: %s",
2353 					strerror(errno));
2354 			return 0;
2355 		}
2356 		log_crypto_err("could not SSL_write");
2357 		return 0;
2358 	}
2359 	sldns_buffer_skip(c->buffer, (ssize_t)r);
2360 	return 1;
2361 #else
2362 	(void)c;
2363 	return 0;
2364 #endif /* HAVE_SSL */
2365 }
2366 
2367 /** write more data for http */
2368 static int
2369 http_write_more(int fd, struct comm_point* c)
2370 {
2371 	ssize_t r;
2372 	log_assert(sldns_buffer_remaining(c->buffer) > 0);
2373 	r = send(fd, (void*)sldns_buffer_current(c->buffer),
2374 		sldns_buffer_remaining(c->buffer), 0);
2375 	if(r == -1) {
2376 #ifndef USE_WINSOCK
2377 		if(errno == EINTR || errno == EAGAIN)
2378 			return 1;
2379 		log_err_addr("http send r", strerror(errno),
2380 			&c->repinfo.addr, c->repinfo.addrlen);
2381 #else
2382 		if(WSAGetLastError() == WSAEINPROGRESS)
2383 			return 1;
2384 		if(WSAGetLastError() == WSAEWOULDBLOCK) {
2385 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2386 			return 1;
2387 		}
2388 		log_err_addr("http send r", wsa_strerror(WSAGetLastError()),
2389 			&c->repinfo.addr, c->repinfo.addrlen);
2390 #endif
2391 		return 0;
2392 	}
2393 	sldns_buffer_skip(c->buffer, r);
2394 	return 1;
2395 }
2396 
2397 /**
2398  * Handle http writing callback.
2399  * @param fd: file descriptor of socket.
2400  * @param c: comm point to write buffer out of.
2401  * @return: 0 on error
2402  */
2403 static int
2404 comm_point_http_handle_write(int fd, struct comm_point* c)
2405 {
2406 	log_assert(c->type == comm_http);
2407 	log_assert(fd != -1);
2408 
2409 	/* check pending connect errors, if that fails, we wait for more,
2410 	 * or we can continue to write contents */
2411 	if(c->tcp_check_nb_connect) {
2412 		int r = http_check_connect(fd, c);
2413 		if(r == 0) return 0;
2414 		if(r == 1) return 1;
2415 		c->tcp_check_nb_connect = 0;
2416 	}
2417 	/* if we are in ssl handshake, handle SSL handshake */
2418 #ifdef HAVE_SSL
2419 	if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
2420 		if(!ssl_handshake(c))
2421 			return 0;
2422 		if(c->ssl_shake_state != comm_ssl_shake_none)
2423 			return 1;
2424 	}
2425 #endif /* HAVE_SSL */
2426 	if(c->tcp_is_reading)
2427 		return 1;
2428 	/* if we are writing, write more */
2429 	if(c->ssl) {
2430 		if(!ssl_http_write_more(c))
2431 			return 0;
2432 	} else {
2433 		if(!http_write_more(fd, c))
2434 			return 0;
2435 	}
2436 
2437 	/* we write a single buffer contents, that can contain
2438 	 * the http request, and then flip to read the results */
2439 	/* see if write is done */
2440 	if(sldns_buffer_remaining(c->buffer) == 0) {
2441 		sldns_buffer_clear(c->buffer);
2442 		if(c->tcp_do_toggle_rw)
2443 			c->tcp_is_reading = 1;
2444 		c->tcp_byte_count = 0;
2445 		/* switch from listening(write) to listening(read) */
2446 		comm_point_stop_listening(c);
2447 		comm_point_start_listening(c, -1, -1);
2448 	}
2449 	return 1;
2450 }
2451 
2452 void
2453 comm_point_http_handle_callback(int fd, short event, void* arg)
2454 {
2455 	struct comm_point* c = (struct comm_point*)arg;
2456 	log_assert(c->type == comm_http);
2457 	ub_comm_base_now(c->ev->base);
2458 
2459 	if(event&UB_EV_TIMEOUT) {
2460 		verbose(VERB_QUERY, "http took too long, dropped");
2461 		reclaim_http_handler(c);
2462 		if(!c->tcp_do_close) {
2463 			fptr_ok(fptr_whitelist_comm_point(c->callback));
2464 			(void)(*c->callback)(c, c->cb_arg,
2465 				NETEVENT_TIMEOUT, NULL);
2466 		}
2467 		return;
2468 	}
2469 	if(event&UB_EV_READ) {
2470 		if(!comm_point_http_handle_read(fd, c)) {
2471 			reclaim_http_handler(c);
2472 			if(!c->tcp_do_close) {
2473 				fptr_ok(fptr_whitelist_comm_point(
2474 					c->callback));
2475 				(void)(*c->callback)(c, c->cb_arg,
2476 					NETEVENT_CLOSED, NULL);
2477 			}
2478 		}
2479 		return;
2480 	}
2481 	if(event&UB_EV_WRITE) {
2482 		if(!comm_point_http_handle_write(fd, c)) {
2483 			reclaim_http_handler(c);
2484 			if(!c->tcp_do_close) {
2485 				fptr_ok(fptr_whitelist_comm_point(
2486 					c->callback));
2487 				(void)(*c->callback)(c, c->cb_arg,
2488 					NETEVENT_CLOSED, NULL);
2489 			}
2490 		}
2491 		return;
2492 	}
2493 	log_err("Ignored event %d for httphdl.", event);
2494 }
2495 
2496 void comm_point_local_handle_callback(int fd, short event, void* arg)
2497 {
2498 	struct comm_point* c = (struct comm_point*)arg;
2499 	log_assert(c->type == comm_local);
2500 	ub_comm_base_now(c->ev->base);
2501 
2502 	if(event&UB_EV_READ) {
2503 		if(!comm_point_tcp_handle_read(fd, c, 1)) {
2504 			fptr_ok(fptr_whitelist_comm_point(c->callback));
2505 			(void)(*c->callback)(c, c->cb_arg, NETEVENT_CLOSED,
2506 				NULL);
2507 		}
2508 		return;
2509 	}
2510 	log_err("Ignored event %d for localhdl.", event);
2511 }
2512 
2513 void comm_point_raw_handle_callback(int ATTR_UNUSED(fd),
2514 	short event, void* arg)
2515 {
2516 	struct comm_point* c = (struct comm_point*)arg;
2517 	int err = NETEVENT_NOERROR;
2518 	log_assert(c->type == comm_raw);
2519 	ub_comm_base_now(c->ev->base);
2520 
2521 	if(event&UB_EV_TIMEOUT)
2522 		err = NETEVENT_TIMEOUT;
2523 	fptr_ok(fptr_whitelist_comm_point_raw(c->callback));
2524 	(void)(*c->callback)(c, c->cb_arg, err, NULL);
2525 }
2526 
2527 struct comm_point*
2528 comm_point_create_udp(struct comm_base *base, int fd, sldns_buffer* buffer,
2529 	comm_point_callback_type* callback, void* callback_arg)
2530 {
2531 	struct comm_point* c = (struct comm_point*)calloc(1,
2532 		sizeof(struct comm_point));
2533 	short evbits;
2534 	if(!c)
2535 		return NULL;
2536 	c->ev = (struct internal_event*)calloc(1,
2537 		sizeof(struct internal_event));
2538 	if(!c->ev) {
2539 		free(c);
2540 		return NULL;
2541 	}
2542 	c->ev->base = base;
2543 	c->fd = fd;
2544 	c->buffer = buffer;
2545 	c->timeout = NULL;
2546 	c->tcp_is_reading = 0;
2547 	c->tcp_byte_count = 0;
2548 	c->tcp_parent = NULL;
2549 	c->max_tcp_count = 0;
2550 	c->cur_tcp_count = 0;
2551 	c->tcp_handlers = NULL;
2552 	c->tcp_free = NULL;
2553 	c->type = comm_udp;
2554 	c->tcp_do_close = 0;
2555 	c->do_not_close = 0;
2556 	c->tcp_do_toggle_rw = 0;
2557 	c->tcp_check_nb_connect = 0;
2558 #ifdef USE_MSG_FASTOPEN
2559 	c->tcp_do_fastopen = 0;
2560 #endif
2561 #ifdef USE_DNSCRYPT
2562 	c->dnscrypt = 0;
2563 	c->dnscrypt_buffer = buffer;
2564 #endif
2565 	c->inuse = 0;
2566 	c->callback = callback;
2567 	c->cb_arg = callback_arg;
2568 	evbits = UB_EV_READ | UB_EV_PERSIST;
2569 	/* ub_event stuff */
2570 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2571 		comm_point_udp_callback, c);
2572 	if(c->ev->ev == NULL) {
2573 		log_err("could not baseset udp event");
2574 		comm_point_delete(c);
2575 		return NULL;
2576 	}
2577 	if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
2578 		log_err("could not add udp event");
2579 		comm_point_delete(c);
2580 		return NULL;
2581 	}
2582 	return c;
2583 }
2584 
2585 struct comm_point*
2586 comm_point_create_udp_ancil(struct comm_base *base, int fd,
2587 	sldns_buffer* buffer,
2588 	comm_point_callback_type* callback, void* callback_arg)
2589 {
2590 	struct comm_point* c = (struct comm_point*)calloc(1,
2591 		sizeof(struct comm_point));
2592 	short evbits;
2593 	if(!c)
2594 		return NULL;
2595 	c->ev = (struct internal_event*)calloc(1,
2596 		sizeof(struct internal_event));
2597 	if(!c->ev) {
2598 		free(c);
2599 		return NULL;
2600 	}
2601 	c->ev->base = base;
2602 	c->fd = fd;
2603 	c->buffer = buffer;
2604 	c->timeout = NULL;
2605 	c->tcp_is_reading = 0;
2606 	c->tcp_byte_count = 0;
2607 	c->tcp_parent = NULL;
2608 	c->max_tcp_count = 0;
2609 	c->cur_tcp_count = 0;
2610 	c->tcp_handlers = NULL;
2611 	c->tcp_free = NULL;
2612 	c->type = comm_udp;
2613 	c->tcp_do_close = 0;
2614 	c->do_not_close = 0;
2615 #ifdef USE_DNSCRYPT
2616 	c->dnscrypt = 0;
2617 	c->dnscrypt_buffer = buffer;
2618 #endif
2619 	c->inuse = 0;
2620 	c->tcp_do_toggle_rw = 0;
2621 	c->tcp_check_nb_connect = 0;
2622 #ifdef USE_MSG_FASTOPEN
2623 	c->tcp_do_fastopen = 0;
2624 #endif
2625 	c->callback = callback;
2626 	c->cb_arg = callback_arg;
2627 	evbits = UB_EV_READ | UB_EV_PERSIST;
2628 	/* ub_event stuff */
2629 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2630 		comm_point_udp_ancil_callback, c);
2631 	if(c->ev->ev == NULL) {
2632 		log_err("could not baseset udp event");
2633 		comm_point_delete(c);
2634 		return NULL;
2635 	}
2636 	if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
2637 		log_err("could not add udp event");
2638 		comm_point_delete(c);
2639 		return NULL;
2640 	}
2641 	return c;
2642 }
2643 
2644 static struct comm_point*
2645 comm_point_create_tcp_handler(struct comm_base *base,
2646 	struct comm_point* parent, size_t bufsize,
2647 	struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
2648 	void* callback_arg)
2649 {
2650 	struct comm_point* c = (struct comm_point*)calloc(1,
2651 		sizeof(struct comm_point));
2652 	short evbits;
2653 	if(!c)
2654 		return NULL;
2655 	c->ev = (struct internal_event*)calloc(1,
2656 		sizeof(struct internal_event));
2657 	if(!c->ev) {
2658 		free(c);
2659 		return NULL;
2660 	}
2661 	c->ev->base = base;
2662 	c->fd = -1;
2663 	c->buffer = sldns_buffer_new(bufsize);
2664 	if(!c->buffer) {
2665 		free(c->ev);
2666 		free(c);
2667 		return NULL;
2668 	}
2669 	c->timeout = (struct timeval*)malloc(sizeof(struct timeval));
2670 	if(!c->timeout) {
2671 		sldns_buffer_free(c->buffer);
2672 		free(c->ev);
2673 		free(c);
2674 		return NULL;
2675 	}
2676 	c->tcp_is_reading = 0;
2677 	c->tcp_byte_count = 0;
2678 	c->tcp_parent = parent;
2679 	c->tcp_timeout_msec = parent->tcp_timeout_msec;
2680 	c->tcp_conn_limit = parent->tcp_conn_limit;
2681 	c->tcl_addr = NULL;
2682 	c->tcp_keepalive = 0;
2683 	c->max_tcp_count = 0;
2684 	c->cur_tcp_count = 0;
2685 	c->tcp_handlers = NULL;
2686 	c->tcp_free = NULL;
2687 	c->type = comm_tcp;
2688 	c->tcp_do_close = 0;
2689 	c->do_not_close = 0;
2690 	c->tcp_do_toggle_rw = 1;
2691 	c->tcp_check_nb_connect = 0;
2692 #ifdef USE_MSG_FASTOPEN
2693 	c->tcp_do_fastopen = 0;
2694 #endif
2695 #ifdef USE_DNSCRYPT
2696 	c->dnscrypt = 0;
2697 	/* We don't know just yet if this is a dnscrypt channel. Allocation
2698 	 * will be done when handling the callback. */
2699 	c->dnscrypt_buffer = c->buffer;
2700 #endif
2701 	c->repinfo.c = c;
2702 	c->callback = callback;
2703 	c->cb_arg = callback_arg;
2704 	if(spoolbuf) {
2705 		c->tcp_req_info = tcp_req_info_create(spoolbuf);
2706 		if(!c->tcp_req_info) {
2707 			log_err("could not create tcp commpoint");
2708 			sldns_buffer_free(c->buffer);
2709 			free(c->timeout);
2710 			free(c->ev);
2711 			free(c);
2712 			return NULL;
2713 		}
2714 		c->tcp_req_info->cp = c;
2715 		c->tcp_do_close = 1;
2716 		c->tcp_do_toggle_rw = 0;
2717 	}
2718 	/* add to parent free list */
2719 	c->tcp_free = parent->tcp_free;
2720 	parent->tcp_free = c;
2721 	/* ub_event stuff */
2722 	evbits = UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT;
2723 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2724 		comm_point_tcp_handle_callback, c);
2725 	if(c->ev->ev == NULL)
2726 	{
2727 		log_err("could not basetset tcphdl event");
2728 		parent->tcp_free = c->tcp_free;
2729 		tcp_req_info_delete(c->tcp_req_info);
2730 		sldns_buffer_free(c->buffer);
2731 		free(c->timeout);
2732 		free(c->ev);
2733 		free(c);
2734 		return NULL;
2735 	}
2736 	return c;
2737 }
2738 
2739 struct comm_point*
2740 comm_point_create_tcp(struct comm_base *base, int fd, int num,
2741 	int idle_timeout, struct tcl_list* tcp_conn_limit, size_t bufsize,
2742 	struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
2743 	void* callback_arg)
2744 {
2745 	struct comm_point* c = (struct comm_point*)calloc(1,
2746 		sizeof(struct comm_point));
2747 	short evbits;
2748 	int i;
2749 	/* first allocate the TCP accept listener */
2750 	if(!c)
2751 		return NULL;
2752 	c->ev = (struct internal_event*)calloc(1,
2753 		sizeof(struct internal_event));
2754 	if(!c->ev) {
2755 		free(c);
2756 		return NULL;
2757 	}
2758 	c->ev->base = base;
2759 	c->fd = fd;
2760 	c->buffer = NULL;
2761 	c->timeout = NULL;
2762 	c->tcp_is_reading = 0;
2763 	c->tcp_byte_count = 0;
2764 	c->tcp_timeout_msec = idle_timeout;
2765 	c->tcp_conn_limit = tcp_conn_limit;
2766 	c->tcl_addr = NULL;
2767 	c->tcp_keepalive = 0;
2768 	c->tcp_parent = NULL;
2769 	c->max_tcp_count = num;
2770 	c->cur_tcp_count = 0;
2771 	c->tcp_handlers = (struct comm_point**)calloc((size_t)num,
2772 		sizeof(struct comm_point*));
2773 	if(!c->tcp_handlers) {
2774 		free(c->ev);
2775 		free(c);
2776 		return NULL;
2777 	}
2778 	c->tcp_free = NULL;
2779 	c->type = comm_tcp_accept;
2780 	c->tcp_do_close = 0;
2781 	c->do_not_close = 0;
2782 	c->tcp_do_toggle_rw = 0;
2783 	c->tcp_check_nb_connect = 0;
2784 #ifdef USE_MSG_FASTOPEN
2785 	c->tcp_do_fastopen = 0;
2786 #endif
2787 #ifdef USE_DNSCRYPT
2788 	c->dnscrypt = 0;
2789 	c->dnscrypt_buffer = NULL;
2790 #endif
2791 	c->callback = NULL;
2792 	c->cb_arg = NULL;
2793 	evbits = UB_EV_READ | UB_EV_PERSIST;
2794 	/* ub_event stuff */
2795 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2796 		comm_point_tcp_accept_callback, c);
2797 	if(c->ev->ev == NULL) {
2798 		log_err("could not baseset tcpacc event");
2799 		comm_point_delete(c);
2800 		return NULL;
2801 	}
2802 	if (ub_event_add(c->ev->ev, c->timeout) != 0) {
2803 		log_err("could not add tcpacc event");
2804 		comm_point_delete(c);
2805 		return NULL;
2806 	}
2807 	/* now prealloc the tcp handlers */
2808 	for(i=0; i<num; i++) {
2809 		c->tcp_handlers[i] = comm_point_create_tcp_handler(base,
2810 			c, bufsize, spoolbuf, callback, callback_arg);
2811 		if(!c->tcp_handlers[i]) {
2812 			comm_point_delete(c);
2813 			return NULL;
2814 		}
2815 	}
2816 
2817 	return c;
2818 }
2819 
2820 struct comm_point*
2821 comm_point_create_tcp_out(struct comm_base *base, size_t bufsize,
2822         comm_point_callback_type* callback, void* callback_arg)
2823 {
2824 	struct comm_point* c = (struct comm_point*)calloc(1,
2825 		sizeof(struct comm_point));
2826 	short evbits;
2827 	if(!c)
2828 		return NULL;
2829 	c->ev = (struct internal_event*)calloc(1,
2830 		sizeof(struct internal_event));
2831 	if(!c->ev) {
2832 		free(c);
2833 		return NULL;
2834 	}
2835 	c->ev->base = base;
2836 	c->fd = -1;
2837 	c->buffer = sldns_buffer_new(bufsize);
2838 	if(!c->buffer) {
2839 		free(c->ev);
2840 		free(c);
2841 		return NULL;
2842 	}
2843 	c->timeout = NULL;
2844 	c->tcp_is_reading = 0;
2845 	c->tcp_byte_count = 0;
2846 	c->tcp_timeout_msec = TCP_QUERY_TIMEOUT;
2847 	c->tcp_conn_limit = NULL;
2848 	c->tcl_addr = NULL;
2849 	c->tcp_keepalive = 0;
2850 	c->tcp_parent = NULL;
2851 	c->max_tcp_count = 0;
2852 	c->cur_tcp_count = 0;
2853 	c->tcp_handlers = NULL;
2854 	c->tcp_free = NULL;
2855 	c->type = comm_tcp;
2856 	c->tcp_do_close = 0;
2857 	c->do_not_close = 0;
2858 	c->tcp_do_toggle_rw = 1;
2859 	c->tcp_check_nb_connect = 1;
2860 #ifdef USE_MSG_FASTOPEN
2861 	c->tcp_do_fastopen = 1;
2862 #endif
2863 #ifdef USE_DNSCRYPT
2864 	c->dnscrypt = 0;
2865 	c->dnscrypt_buffer = c->buffer;
2866 #endif
2867 	c->repinfo.c = c;
2868 	c->callback = callback;
2869 	c->cb_arg = callback_arg;
2870 	evbits = UB_EV_PERSIST | UB_EV_WRITE;
2871 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2872 		comm_point_tcp_handle_callback, c);
2873 	if(c->ev->ev == NULL)
2874 	{
2875 		log_err("could not baseset tcpout event");
2876 		sldns_buffer_free(c->buffer);
2877 		free(c->ev);
2878 		free(c);
2879 		return NULL;
2880 	}
2881 
2882 	return c;
2883 }
2884 
2885 struct comm_point*
2886 comm_point_create_http_out(struct comm_base *base, size_t bufsize,
2887         comm_point_callback_type* callback, void* callback_arg,
2888 	sldns_buffer* temp)
2889 {
2890 	struct comm_point* c = (struct comm_point*)calloc(1,
2891 		sizeof(struct comm_point));
2892 	short evbits;
2893 	if(!c)
2894 		return NULL;
2895 	c->ev = (struct internal_event*)calloc(1,
2896 		sizeof(struct internal_event));
2897 	if(!c->ev) {
2898 		free(c);
2899 		return NULL;
2900 	}
2901 	c->ev->base = base;
2902 	c->fd = -1;
2903 	c->buffer = sldns_buffer_new(bufsize);
2904 	if(!c->buffer) {
2905 		free(c->ev);
2906 		free(c);
2907 		return NULL;
2908 	}
2909 	c->timeout = NULL;
2910 	c->tcp_is_reading = 0;
2911 	c->tcp_byte_count = 0;
2912 	c->tcp_parent = NULL;
2913 	c->max_tcp_count = 0;
2914 	c->cur_tcp_count = 0;
2915 	c->tcp_handlers = NULL;
2916 	c->tcp_free = NULL;
2917 	c->type = comm_http;
2918 	c->tcp_do_close = 0;
2919 	c->do_not_close = 0;
2920 	c->tcp_do_toggle_rw = 1;
2921 	c->tcp_check_nb_connect = 1;
2922 	c->http_in_headers = 1;
2923 	c->http_in_chunk_headers = 0;
2924 	c->http_is_chunked = 0;
2925 	c->http_temp = temp;
2926 #ifdef USE_MSG_FASTOPEN
2927 	c->tcp_do_fastopen = 1;
2928 #endif
2929 #ifdef USE_DNSCRYPT
2930 	c->dnscrypt = 0;
2931 	c->dnscrypt_buffer = c->buffer;
2932 #endif
2933 	c->repinfo.c = c;
2934 	c->callback = callback;
2935 	c->cb_arg = callback_arg;
2936 	evbits = UB_EV_PERSIST | UB_EV_WRITE;
2937 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2938 		comm_point_http_handle_callback, c);
2939 	if(c->ev->ev == NULL)
2940 	{
2941 		log_err("could not baseset tcpout event");
2942 #ifdef HAVE_SSL
2943 		SSL_free(c->ssl);
2944 #endif
2945 		sldns_buffer_free(c->buffer);
2946 		free(c->ev);
2947 		free(c);
2948 		return NULL;
2949 	}
2950 
2951 	return c;
2952 }
2953 
2954 struct comm_point*
2955 comm_point_create_local(struct comm_base *base, int fd, size_t bufsize,
2956         comm_point_callback_type* callback, void* callback_arg)
2957 {
2958 	struct comm_point* c = (struct comm_point*)calloc(1,
2959 		sizeof(struct comm_point));
2960 	short evbits;
2961 	if(!c)
2962 		return NULL;
2963 	c->ev = (struct internal_event*)calloc(1,
2964 		sizeof(struct internal_event));
2965 	if(!c->ev) {
2966 		free(c);
2967 		return NULL;
2968 	}
2969 	c->ev->base = base;
2970 	c->fd = fd;
2971 	c->buffer = sldns_buffer_new(bufsize);
2972 	if(!c->buffer) {
2973 		free(c->ev);
2974 		free(c);
2975 		return NULL;
2976 	}
2977 	c->timeout = NULL;
2978 	c->tcp_is_reading = 1;
2979 	c->tcp_byte_count = 0;
2980 	c->tcp_parent = NULL;
2981 	c->max_tcp_count = 0;
2982 	c->cur_tcp_count = 0;
2983 	c->tcp_handlers = NULL;
2984 	c->tcp_free = NULL;
2985 	c->type = comm_local;
2986 	c->tcp_do_close = 0;
2987 	c->do_not_close = 1;
2988 	c->tcp_do_toggle_rw = 0;
2989 	c->tcp_check_nb_connect = 0;
2990 #ifdef USE_MSG_FASTOPEN
2991 	c->tcp_do_fastopen = 0;
2992 #endif
2993 #ifdef USE_DNSCRYPT
2994 	c->dnscrypt = 0;
2995 	c->dnscrypt_buffer = c->buffer;
2996 #endif
2997 	c->callback = callback;
2998 	c->cb_arg = callback_arg;
2999 	/* ub_event stuff */
3000 	evbits = UB_EV_PERSIST | UB_EV_READ;
3001 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3002 		comm_point_local_handle_callback, c);
3003 	if(c->ev->ev == NULL) {
3004 		log_err("could not baseset localhdl event");
3005 		free(c->ev);
3006 		free(c);
3007 		return NULL;
3008 	}
3009 	if (ub_event_add(c->ev->ev, c->timeout) != 0) {
3010 		log_err("could not add localhdl event");
3011 		ub_event_free(c->ev->ev);
3012 		free(c->ev);
3013 		free(c);
3014 		return NULL;
3015 	}
3016 	return c;
3017 }
3018 
3019 struct comm_point*
3020 comm_point_create_raw(struct comm_base* base, int fd, int writing,
3021 	comm_point_callback_type* callback, void* callback_arg)
3022 {
3023 	struct comm_point* c = (struct comm_point*)calloc(1,
3024 		sizeof(struct comm_point));
3025 	short evbits;
3026 	if(!c)
3027 		return NULL;
3028 	c->ev = (struct internal_event*)calloc(1,
3029 		sizeof(struct internal_event));
3030 	if(!c->ev) {
3031 		free(c);
3032 		return NULL;
3033 	}
3034 	c->ev->base = base;
3035 	c->fd = fd;
3036 	c->buffer = NULL;
3037 	c->timeout = NULL;
3038 	c->tcp_is_reading = 0;
3039 	c->tcp_byte_count = 0;
3040 	c->tcp_parent = NULL;
3041 	c->max_tcp_count = 0;
3042 	c->cur_tcp_count = 0;
3043 	c->tcp_handlers = NULL;
3044 	c->tcp_free = NULL;
3045 	c->type = comm_raw;
3046 	c->tcp_do_close = 0;
3047 	c->do_not_close = 1;
3048 	c->tcp_do_toggle_rw = 0;
3049 	c->tcp_check_nb_connect = 0;
3050 #ifdef USE_MSG_FASTOPEN
3051 	c->tcp_do_fastopen = 0;
3052 #endif
3053 #ifdef USE_DNSCRYPT
3054 	c->dnscrypt = 0;
3055 	c->dnscrypt_buffer = c->buffer;
3056 #endif
3057 	c->callback = callback;
3058 	c->cb_arg = callback_arg;
3059 	/* ub_event stuff */
3060 	if(writing)
3061 		evbits = UB_EV_PERSIST | UB_EV_WRITE;
3062 	else 	evbits = UB_EV_PERSIST | UB_EV_READ;
3063 	c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3064 		comm_point_raw_handle_callback, c);
3065 	if(c->ev->ev == NULL) {
3066 		log_err("could not baseset rawhdl event");
3067 		free(c->ev);
3068 		free(c);
3069 		return NULL;
3070 	}
3071 	if (ub_event_add(c->ev->ev, c->timeout) != 0) {
3072 		log_err("could not add rawhdl event");
3073 		ub_event_free(c->ev->ev);
3074 		free(c->ev);
3075 		free(c);
3076 		return NULL;
3077 	}
3078 	return c;
3079 }
3080 
3081 void
3082 comm_point_close(struct comm_point* c)
3083 {
3084 	if(!c)
3085 		return;
3086 	if(c->fd != -1) {
3087 		if(ub_event_del(c->ev->ev) != 0) {
3088 			log_err("could not event_del on close");
3089 		}
3090 	}
3091 	tcl_close_connection(c->tcl_addr);
3092 	if(c->tcp_req_info)
3093 		tcp_req_info_clear(c->tcp_req_info);
3094 	/* close fd after removing from event lists, or epoll.. is messed up */
3095 	if(c->fd != -1 && !c->do_not_close) {
3096 		if(c->type == comm_tcp || c->type == comm_http) {
3097 			/* delete sticky events for the fd, it gets closed */
3098 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
3099 			ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
3100 		}
3101 		verbose(VERB_ALGO, "close fd %d", c->fd);
3102 #ifndef USE_WINSOCK
3103 		close(c->fd);
3104 #else
3105 		closesocket(c->fd);
3106 #endif
3107 	}
3108 	c->fd = -1;
3109 }
3110 
3111 void
3112 comm_point_delete(struct comm_point* c)
3113 {
3114 	if(!c)
3115 		return;
3116 	if((c->type == comm_tcp || c->type == comm_http) && c->ssl) {
3117 #ifdef HAVE_SSL
3118 		SSL_shutdown(c->ssl);
3119 		SSL_free(c->ssl);
3120 #endif
3121 	}
3122 	comm_point_close(c);
3123 	if(c->tcp_handlers) {
3124 		int i;
3125 		for(i=0; i<c->max_tcp_count; i++)
3126 			comm_point_delete(c->tcp_handlers[i]);
3127 		free(c->tcp_handlers);
3128 	}
3129 	free(c->timeout);
3130 	if(c->type == comm_tcp || c->type == comm_local || c->type == comm_http) {
3131 		sldns_buffer_free(c->buffer);
3132 #ifdef USE_DNSCRYPT
3133 		if(c->dnscrypt && c->dnscrypt_buffer != c->buffer) {
3134 			sldns_buffer_free(c->dnscrypt_buffer);
3135 		}
3136 #endif
3137 		if(c->tcp_req_info) {
3138 			tcp_req_info_delete(c->tcp_req_info);
3139 		}
3140 	}
3141 	ub_event_free(c->ev->ev);
3142 	free(c->ev);
3143 	free(c);
3144 }
3145 
3146 void
3147 comm_point_send_reply(struct comm_reply *repinfo)
3148 {
3149 	struct sldns_buffer* buffer;
3150 	log_assert(repinfo && repinfo->c);
3151 #ifdef USE_DNSCRYPT
3152 	buffer = repinfo->c->dnscrypt_buffer;
3153 	if(!dnsc_handle_uncurved_request(repinfo)) {
3154 		return;
3155 	}
3156 #else
3157 	buffer = repinfo->c->buffer;
3158 #endif
3159 	if(repinfo->c->type == comm_udp) {
3160 		if(repinfo->srctype)
3161 			comm_point_send_udp_msg_if(repinfo->c,
3162 			buffer, (struct sockaddr*)&repinfo->addr,
3163 			repinfo->addrlen, repinfo);
3164 		else
3165 			comm_point_send_udp_msg(repinfo->c, buffer,
3166 			(struct sockaddr*)&repinfo->addr, repinfo->addrlen);
3167 #ifdef USE_DNSTAP
3168 		if(repinfo->c->dtenv != NULL &&
3169 		   repinfo->c->dtenv->log_client_response_messages)
3170 			dt_msg_send_client_response(repinfo->c->dtenv,
3171 			&repinfo->addr, repinfo->c->type, repinfo->c->buffer);
3172 #endif
3173 	} else {
3174 #ifdef USE_DNSTAP
3175 		if(repinfo->c->tcp_parent->dtenv != NULL &&
3176 		   repinfo->c->tcp_parent->dtenv->log_client_response_messages)
3177 			dt_msg_send_client_response(repinfo->c->tcp_parent->dtenv,
3178 			&repinfo->addr, repinfo->c->type, repinfo->c->buffer);
3179 #endif
3180 		if(repinfo->c->tcp_req_info) {
3181 			tcp_req_info_send_reply(repinfo->c->tcp_req_info);
3182 		} else {
3183 			comm_point_start_listening(repinfo->c, -1,
3184 				repinfo->c->tcp_timeout_msec);
3185 		}
3186 	}
3187 }
3188 
3189 void
3190 comm_point_drop_reply(struct comm_reply* repinfo)
3191 {
3192 	if(!repinfo)
3193 		return;
3194 	log_assert(repinfo->c);
3195 	log_assert(repinfo->c->type != comm_tcp_accept);
3196 	if(repinfo->c->type == comm_udp)
3197 		return;
3198 	if(repinfo->c->tcp_req_info)
3199 		repinfo->c->tcp_req_info->is_drop = 1;
3200 	reclaim_tcp_handler(repinfo->c);
3201 }
3202 
3203 void
3204 comm_point_stop_listening(struct comm_point* c)
3205 {
3206 	verbose(VERB_ALGO, "comm point stop listening %d", c->fd);
3207 	if(ub_event_del(c->ev->ev) != 0) {
3208 		log_err("event_del error to stoplisten");
3209 	}
3210 }
3211 
3212 void
3213 comm_point_start_listening(struct comm_point* c, int newfd, int msec)
3214 {
3215 	verbose(VERB_ALGO, "comm point start listening %d (%d msec)",
3216 		c->fd==-1?newfd:c->fd, msec);
3217 	if(c->type == comm_tcp_accept && !c->tcp_free) {
3218 		/* no use to start listening no free slots. */
3219 		return;
3220 	}
3221 	if(msec != -1 && msec != 0) {
3222 		if(!c->timeout) {
3223 			c->timeout = (struct timeval*)malloc(sizeof(
3224 				struct timeval));
3225 			if(!c->timeout) {
3226 				log_err("cpsl: malloc failed. No net read.");
3227 				return;
3228 			}
3229 		}
3230 		ub_event_add_bits(c->ev->ev, UB_EV_TIMEOUT);
3231 #ifndef S_SPLINT_S /* splint fails on struct timeval. */
3232 		c->timeout->tv_sec = msec/1000;
3233 		c->timeout->tv_usec = (msec%1000)*1000;
3234 #endif /* S_SPLINT_S */
3235 	}
3236 	if(c->type == comm_tcp || c->type == comm_http) {
3237 		ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
3238 		if(c->tcp_is_reading)
3239 			ub_event_add_bits(c->ev->ev, UB_EV_READ);
3240 		else	ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
3241 	}
3242 	if(newfd != -1) {
3243 		if(c->fd != -1) {
3244 #ifndef USE_WINSOCK
3245 			close(c->fd);
3246 #else
3247 			closesocket(c->fd);
3248 #endif
3249 		}
3250 		c->fd = newfd;
3251 		ub_event_set_fd(c->ev->ev, c->fd);
3252 	}
3253 	if(ub_event_add(c->ev->ev, msec==0?NULL:c->timeout) != 0) {
3254 		log_err("event_add failed. in cpsl.");
3255 	}
3256 }
3257 
3258 void comm_point_listen_for_rw(struct comm_point* c, int rd, int wr)
3259 {
3260 	verbose(VERB_ALGO, "comm point listen_for_rw %d %d", c->fd, wr);
3261 	if(ub_event_del(c->ev->ev) != 0) {
3262 		log_err("event_del error to cplf");
3263 	}
3264 	ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
3265 	if(rd) ub_event_add_bits(c->ev->ev, UB_EV_READ);
3266 	if(wr) ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
3267 	if(ub_event_add(c->ev->ev, c->timeout) != 0) {
3268 		log_err("event_add failed. in cplf.");
3269 	}
3270 }
3271 
3272 size_t comm_point_get_mem(struct comm_point* c)
3273 {
3274 	size_t s;
3275 	if(!c)
3276 		return 0;
3277 	s = sizeof(*c) + sizeof(*c->ev);
3278 	if(c->timeout)
3279 		s += sizeof(*c->timeout);
3280 	if(c->type == comm_tcp || c->type == comm_local) {
3281 		s += sizeof(*c->buffer) + sldns_buffer_capacity(c->buffer);
3282 #ifdef USE_DNSCRYPT
3283 		s += sizeof(*c->dnscrypt_buffer);
3284 		if(c->buffer != c->dnscrypt_buffer) {
3285 			s += sldns_buffer_capacity(c->dnscrypt_buffer);
3286 		}
3287 #endif
3288 	}
3289 	if(c->type == comm_tcp_accept) {
3290 		int i;
3291 		for(i=0; i<c->max_tcp_count; i++)
3292 			s += comm_point_get_mem(c->tcp_handlers[i]);
3293 	}
3294 	return s;
3295 }
3296 
3297 struct comm_timer*
3298 comm_timer_create(struct comm_base* base, void (*cb)(void*), void* cb_arg)
3299 {
3300 	struct internal_timer *tm = (struct internal_timer*)calloc(1,
3301 		sizeof(struct internal_timer));
3302 	if(!tm) {
3303 		log_err("malloc failed");
3304 		return NULL;
3305 	}
3306 	tm->super.ev_timer = tm;
3307 	tm->base = base;
3308 	tm->super.callback = cb;
3309 	tm->super.cb_arg = cb_arg;
3310 	tm->ev = ub_event_new(base->eb->base, -1, UB_EV_TIMEOUT,
3311 		comm_timer_callback, &tm->super);
3312 	if(tm->ev == NULL) {
3313 		log_err("timer_create: event_base_set failed.");
3314 		free(tm);
3315 		return NULL;
3316 	}
3317 	return &tm->super;
3318 }
3319 
3320 void
3321 comm_timer_disable(struct comm_timer* timer)
3322 {
3323 	if(!timer)
3324 		return;
3325 	ub_timer_del(timer->ev_timer->ev);
3326 	timer->ev_timer->enabled = 0;
3327 }
3328 
3329 void
3330 comm_timer_set(struct comm_timer* timer, struct timeval* tv)
3331 {
3332 	log_assert(tv);
3333 	if(timer->ev_timer->enabled)
3334 		comm_timer_disable(timer);
3335 	if(ub_timer_add(timer->ev_timer->ev, timer->ev_timer->base->eb->base,
3336 		comm_timer_callback, timer, tv) != 0)
3337 		log_err("comm_timer_set: evtimer_add failed.");
3338 	timer->ev_timer->enabled = 1;
3339 }
3340 
3341 void
3342 comm_timer_delete(struct comm_timer* timer)
3343 {
3344 	if(!timer)
3345 		return;
3346 	comm_timer_disable(timer);
3347 	/* Free the sub struct timer->ev_timer derived from the super struct timer.
3348 	 * i.e. assert(timer == timer->ev_timer)
3349 	 */
3350 	ub_event_free(timer->ev_timer->ev);
3351 	free(timer->ev_timer);
3352 }
3353 
3354 void
3355 comm_timer_callback(int ATTR_UNUSED(fd), short event, void* arg)
3356 {
3357 	struct comm_timer* tm = (struct comm_timer*)arg;
3358 	if(!(event&UB_EV_TIMEOUT))
3359 		return;
3360 	ub_comm_base_now(tm->ev_timer->base);
3361 	tm->ev_timer->enabled = 0;
3362 	fptr_ok(fptr_whitelist_comm_timer(tm->callback));
3363 	(*tm->callback)(tm->cb_arg);
3364 }
3365 
3366 int
3367 comm_timer_is_set(struct comm_timer* timer)
3368 {
3369 	return (int)timer->ev_timer->enabled;
3370 }
3371 
3372 size_t
3373 comm_timer_get_mem(struct comm_timer* ATTR_UNUSED(timer))
3374 {
3375 	return sizeof(struct internal_timer);
3376 }
3377 
3378 struct comm_signal*
3379 comm_signal_create(struct comm_base* base,
3380         void (*callback)(int, void*), void* cb_arg)
3381 {
3382 	struct comm_signal* com = (struct comm_signal*)malloc(
3383 		sizeof(struct comm_signal));
3384 	if(!com) {
3385 		log_err("malloc failed");
3386 		return NULL;
3387 	}
3388 	com->base = base;
3389 	com->callback = callback;
3390 	com->cb_arg = cb_arg;
3391 	com->ev_signal = NULL;
3392 	return com;
3393 }
3394 
3395 void
3396 comm_signal_callback(int sig, short event, void* arg)
3397 {
3398 	struct comm_signal* comsig = (struct comm_signal*)arg;
3399 	if(!(event & UB_EV_SIGNAL))
3400 		return;
3401 	ub_comm_base_now(comsig->base);
3402 	fptr_ok(fptr_whitelist_comm_signal(comsig->callback));
3403 	(*comsig->callback)(sig, comsig->cb_arg);
3404 }
3405 
3406 int
3407 comm_signal_bind(struct comm_signal* comsig, int sig)
3408 {
3409 	struct internal_signal* entry = (struct internal_signal*)calloc(1,
3410 		sizeof(struct internal_signal));
3411 	if(!entry) {
3412 		log_err("malloc failed");
3413 		return 0;
3414 	}
3415 	log_assert(comsig);
3416 	/* add signal event */
3417 	entry->ev = ub_signal_new(comsig->base->eb->base, sig,
3418 		comm_signal_callback, comsig);
3419 	if(entry->ev == NULL) {
3420 		log_err("Could not create signal event");
3421 		free(entry);
3422 		return 0;
3423 	}
3424 	if(ub_signal_add(entry->ev, NULL) != 0) {
3425 		log_err("Could not add signal handler");
3426 		ub_event_free(entry->ev);
3427 		free(entry);
3428 		return 0;
3429 	}
3430 	/* link into list */
3431 	entry->next = comsig->ev_signal;
3432 	comsig->ev_signal = entry;
3433 	return 1;
3434 }
3435 
3436 void
3437 comm_signal_delete(struct comm_signal* comsig)
3438 {
3439 	struct internal_signal* p, *np;
3440 	if(!comsig)
3441 		return;
3442 	p=comsig->ev_signal;
3443 	while(p) {
3444 		np = p->next;
3445 		ub_signal_del(p->ev);
3446 		ub_event_free(p->ev);
3447 		free(p);
3448 		p = np;
3449 	}
3450 	free(comsig);
3451 }
3452