xref: /freebsd/contrib/unbound/testcode/doqclient.c (revision b670c9bafc0e31c7609969bf374b2e80bdc00211)
1 /*
2  * testcode/doqclient.c - debug program. Perform multiple DNS queries using DoQ.
3  *
4  * Copyright (c) 2022, NLnet Labs. All rights reserved.
5  *
6  * This software is open source.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright notice,
13  * this list of conditions and the following disclaimer.
14  *
15  * Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  *
19  * Neither the name of the NLNET LABS nor the names of its contributors may
20  * be used to endorse or promote products derived from this software without
21  * specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 /**
37  * \file
38  *
39  * Simple DNS-over-QUIC client. For testing and debugging purposes.
40  * No authentication of TLS cert.
41  */
42 
43 #include "config.h"
44 #ifdef HAVE_GETOPT_H
45 #include <getopt.h>
46 #endif
47 
48 #ifdef HAVE_NGTCP2
49 #include <ngtcp2/ngtcp2.h>
50 #include <ngtcp2/ngtcp2_crypto.h>
51 #ifdef HAVE_NGTCP2_NGTCP2_CRYPTO_QUICTLS_H
52 #include <ngtcp2/ngtcp2_crypto_quictls.h>
53 #else
54 #include <ngtcp2/ngtcp2_crypto_openssl.h>
55 #endif
56 #include <openssl/ssl.h>
57 #include <openssl/rand.h>
58 #ifdef HAVE_TIME_H
59 #include <time.h>
60 #endif
61 #include <sys/time.h>
62 #include "util/locks.h"
63 #include "util/net_help.h"
64 #include "sldns/sbuffer.h"
65 #include "sldns/str2wire.h"
66 #include "sldns/wire2str.h"
67 #include "util/data/msgreply.h"
68 #include "util/data/msgencode.h"
69 #include "util/data/msgparse.h"
70 #include "util/data/dname.h"
71 #include "util/random.h"
72 #include "util/ub_event.h"
73 struct doq_client_stream_list;
74 struct doq_client_stream;
75 
76 /** the local client data for the DoQ connection */
77 struct doq_client_data {
78 	/** file descriptor */
79 	int fd;
80 	/** the event base for the events */
81 	struct ub_event_base* base;
82 	/** the ub event */
83 	struct ub_event* ev;
84 	/** the expiry timer */
85 	struct ub_event* expire_timer;
86 	/** is the expire_timer added */
87 	int expire_timer_added;
88 	/** the ngtcp2 connection information */
89 	struct ngtcp2_conn* conn;
90 	/** random state */
91 	struct ub_randstate* rnd;
92 	/** server connected to as a string */
93 	const char* svr;
94 	/** the static secret */
95 	uint8_t* static_secret_data;
96 	/** the static secret size */
97 	size_t static_secret_size;
98 	/** destination address sockaddr */
99 	struct sockaddr_storage dest_addr;
100 	/** length of dest addr */
101 	socklen_t dest_addr_len;
102 	/** local address sockaddr */
103 	struct sockaddr_storage local_addr;
104 	/** length of local addr */
105 	socklen_t local_addr_len;
106 	/** SSL context */
107 	SSL_CTX* ctx;
108 	/** SSL object */
109 	SSL* ssl;
110 #ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
111 	/** the connection reference for ngtcp2_conn and userdata in ssl */
112 	struct ngtcp2_crypto_conn_ref conn_ref;
113 #endif
114 	/** the quic version to use */
115 	uint32_t quic_version;
116 	/** the last error */
117 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
118 	struct ngtcp2_ccerr ccerr;
119 #else
120 	struct ngtcp2_connection_close_error last_error;
121 #endif
122 	/** the recent tls alert error code */
123 	uint8_t tls_alert;
124 	/** the buffer for packet operations */
125 	struct sldns_buffer* pkt_buf;
126 	/** The list of queries to start. They have no stream associated.
127 	 * Once they do, they move to the send list. */
128 	struct doq_client_stream_list* query_list_start;
129 	/** The list of queries to send. They have a stream, and they are
130 	 * sending data. Data could also be received, like errors. */
131 	struct doq_client_stream_list* query_list_send;
132 	/** The list of queries to receive. They have a stream, and the
133 	 * send is done, it is possible to read data. */
134 	struct doq_client_stream_list* query_list_receive;
135 	/** The list of queries that are stopped. They have no stream
136 	 * active any more. Write and read are done. The query is done,
137 	 * and it may be in error and then have no answer or partial answer. */
138 	struct doq_client_stream_list* query_list_stop;
139 	/** is there a blocked packet in the blocked_pkt buffer */
140 	int have_blocked_pkt;
141 	/** store blocked packet, a packet that could not be sent on the
142 	 * nonblocking socket. */
143 	struct sldns_buffer* blocked_pkt;
144 	/** ecn info for the blocked packet */
145 	struct ngtcp2_pkt_info blocked_pkt_pi;
146 	/** the congestion control algorithm */
147 	ngtcp2_cc_algo cc_algo;
148 	/** the transport parameters file, for early data transmission */
149 	const char* transport_file;
150 	/** the tls session file, for session resumption */
151 	const char* session_file;
152 	/** if early data is enabled for the connection */
153 	int early_data_enabled;
154 	/** how quiet is the output */
155 	int quiet;
156 	/** the configured port for the destination */
157 	int port;
158 };
159 
160 /** the local client stream list, for appending streams to */
161 struct doq_client_stream_list {
162 	/** first and last members of the list */
163 	struct doq_client_stream* first, *last;
164 };
165 
166 /** the local client data for a DoQ stream */
167 struct doq_client_stream {
168 	/** next stream in list, and prev in list */
169 	struct doq_client_stream* next, *prev;
170 	/** the data buffer */
171 	uint8_t* data;
172 	/** length of the data buffer */
173 	size_t data_len;
174 	/** if the client query has a stream, that is active, associated with
175 	 * it. The stream_id is in stream_id. */
176 	int has_stream;
177 	/** the stream id */
178 	int64_t stream_id;
179 	/** data written position */
180 	size_t nwrite;
181 	/** the data length for write, in network format */
182 	uint16_t data_tcplen;
183 	/** if the write of the query data is done. That means the
184 	 * write channel has FIN, is closed for writing. */
185 	int write_is_done;
186 	/** data read position */
187 	size_t nread;
188 	/** the answer length, in network byte order */
189 	uint16_t answer_len;
190 	/** the answer buffer */
191 	struct sldns_buffer* answer;
192 	/** the answer is complete */
193 	int answer_is_complete;
194 	/** the query has an error, it has no answer, or no complete answer */
195 	int query_has_error;
196 	/** if the query is done */
197 	int query_is_done;
198 };
199 
200 #ifndef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
201 /** the quic method struct, must remain valid during the QUIC connection. */
202 static SSL_QUIC_METHOD quic_method;
203 #endif
204 
205 /** Get the connection ngtcp2_conn from the ssl app data
206  * ngtcp2_crypto_conn_ref */
207 static ngtcp2_conn* conn_ref_get_conn(ngtcp2_crypto_conn_ref* conn_ref)
208 {
209 	struct doq_client_data* data = (struct doq_client_data*)
210 		conn_ref->user_data;
211 	return data->conn;
212 }
213 
214 static void
215 set_app_data(SSL* ssl, struct doq_client_data* data)
216 {
217 #ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
218 	data->conn_ref.get_conn = &conn_ref_get_conn;
219 	data->conn_ref.user_data = data;
220 	SSL_set_app_data(ssl, &data->conn_ref);
221 #else
222 	SSL_set_app_data(ssl, data);
223 #endif
224 }
225 
226 static struct doq_client_data*
227 get_app_data(SSL* ssl)
228 {
229 	struct doq_client_data* data;
230 #ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
231 	data = (struct doq_client_data*)((struct ngtcp2_crypto_conn_ref*)
232 		SSL_get_app_data(ssl))->user_data;
233 #else
234 	data = (struct doq_client_data*) SSL_get_app_data(ssl);
235 #endif
236 	return data;
237 }
238 
239 
240 
241 /** write handle routine */
242 static void on_write(struct doq_client_data* data);
243 /** update the timer */
244 static void update_timer(struct doq_client_data* data);
245 /** disconnect we are done */
246 static void disconnect(struct doq_client_data* data);
247 /** fetch and write the transport file */
248 static void early_data_write_transport(struct doq_client_data* data);
249 
250 /** usage of doqclient */
251 static void usage(char* argv[])
252 {
253 	printf("usage: %s [options] name type class ...\n", argv[0]);
254 	printf("	sends the name-type-class queries over "
255 			"DNS-over-QUIC.\n");
256 	printf("-s server	IP address to send the queries to, "
257 			"default: 127.0.0.1\n");
258 	printf("-p		Port to connect to, default: %d\n",
259 		UNBOUND_DNS_OVER_QUIC_PORT);
260 	printf("-v 		verbose output\n");
261 	printf("-q 		quiet, short output of answer\n");
262 	printf("-x file		transport file, for read/write of transport parameters.\n\t\tIf it exists, it is used to send early data. It is then\n\t\twritten to contain the last used transport parameters.\n\t\tAlso -y must be enabled for early data to succeed.\n");
263 	printf("-y file		session file, for read/write of TLS session. If it exists,\n\t\tit is used for TLS session resumption. It is then written\n\t\tto contain the last session used.\n\t\tOn its own, without also -x, resumes TLS session.\n");
264 	printf("-h 		This help text\n");
265 	exit(1);
266 }
267 
268 /** get the dest address */
269 static void
270 get_dest_addr(struct doq_client_data* data, const char* svr, int port)
271 {
272 	if(!ipstrtoaddr(svr, port, &data->dest_addr, &data->dest_addr_len)) {
273 		printf("fatal: bad server specs '%s'\n", svr);
274 		exit(1);
275 	}
276 }
277 
278 /** open UDP socket to svr */
279 static int
280 open_svr_udp(struct doq_client_data* data)
281 {
282 	int fd = -1;
283 	int r;
284 	fd = socket(addr_is_ip6(&data->dest_addr, data->dest_addr_len)?
285 		PF_INET6:PF_INET, SOCK_DGRAM, 0);
286 	if(fd == -1) {
287 		perror("socket() error");
288 		exit(1);
289 	}
290 	r = connect(fd, (struct sockaddr*)&data->dest_addr,
291 		data->dest_addr_len);
292 	if(r < 0 && r != EINPROGRESS) {
293 		perror("connect() error");
294 		exit(1);
295 	}
296 	fd_set_nonblock(fd);
297 	return fd;
298 }
299 
300 /** get the local address of the connection */
301 static void
302 get_local_addr(struct doq_client_data* data)
303 {
304 	memset(&data->local_addr, 0, sizeof(data->local_addr));
305 	data->local_addr_len = (socklen_t)sizeof(data->local_addr);
306 	if(getsockname(data->fd, (struct sockaddr*)&data->local_addr,
307 		&data->local_addr_len) == -1) {
308 		perror("getsockname() error");
309 		exit(1);
310 	}
311 	log_addr(1, "local_addr", &data->local_addr, data->local_addr_len);
312 	log_addr(1, "dest_addr", &data->dest_addr, data->dest_addr_len);
313 }
314 
315 static sldns_buffer*
316 make_query(char* qname, char* qtype, char* qclass)
317 {
318 	struct query_info qinfo;
319 	struct edns_data edns;
320 	sldns_buffer* buf = sldns_buffer_new(65553);
321 	if(!buf) fatal_exit("out of memory");
322 	qinfo.qname = sldns_str2wire_dname(qname, &qinfo.qname_len);
323 	if(!qinfo.qname) {
324 		printf("cannot parse query name: '%s'\n", qname);
325 		exit(1);
326 	}
327 
328 	qinfo.qtype = sldns_get_rr_type_by_name(qtype);
329 	qinfo.qclass = sldns_get_rr_class_by_name(qclass);
330 	qinfo.local_alias = NULL;
331 
332 	qinfo_query_encode(buf, &qinfo); /* flips buffer */
333 	free(qinfo.qname);
334 	sldns_buffer_write_u16_at(buf, 0, 0x0000);
335 	sldns_buffer_write_u16_at(buf, 2, BIT_RD);
336 	memset(&edns, 0, sizeof(edns));
337 	edns.edns_present = 1;
338 	edns.bits = EDNS_DO;
339 	edns.udp_size = 4096;
340 	if(sldns_buffer_capacity(buf) >=
341 		sldns_buffer_limit(buf)+calc_edns_field_size(&edns))
342 		attach_edns_record(buf, &edns);
343 	return buf;
344 }
345 
346 /** create client stream structure */
347 static struct doq_client_stream*
348 client_stream_create(struct sldns_buffer* query_data)
349 {
350 	struct doq_client_stream* str = calloc(1, sizeof(*str));
351 	if(!str)
352 		fatal_exit("calloc failed: out of memory");
353 	str->data = memdup(sldns_buffer_begin(query_data),
354 		sldns_buffer_limit(query_data));
355 	if(!str->data)
356 		fatal_exit("alloc data failed: out of memory");
357 	str->data_len = sldns_buffer_limit(query_data);
358 	str->stream_id = -1;
359 	return str;
360 }
361 
362 /** free client stream structure */
363 static void
364 client_stream_free(struct doq_client_stream* str)
365 {
366 	if(!str)
367 		return;
368 	free(str->data);
369 	sldns_buffer_free(str->answer);
370 	free(str);
371 }
372 
373 /** setup the stream to start the write process */
374 static void
375 client_stream_start_setup(struct doq_client_stream* str, int64_t stream_id)
376 {
377 	str->has_stream = 1;
378 	str->stream_id = stream_id;
379 	str->nwrite = 0;
380 	str->nread = 0;
381 	str->answer_len = 0;
382 	str->query_is_done = 0;
383 	str->answer_is_complete = 0;
384 	str->query_has_error = 0;
385 	if(str->answer) {
386 		sldns_buffer_free(str->answer);
387 		str->answer = NULL;
388 	}
389 }
390 
391 /** Return string for log purposes with query name. */
392 static char*
393 client_stream_string(struct doq_client_stream* str)
394 {
395 	char* s;
396 	size_t dname_len;
397 	char dname[256], tpstr[32], result[256+32+16];
398 	uint16_t tp;
399 	if(str->data_len <= LDNS_HEADER_SIZE) {
400 		s = strdup("query_with_no_question");
401 		if(!s)
402 			fatal_exit("strdup failed: out of memory");
403 		return s;
404 	}
405 	dname_len = dname_valid(str->data+LDNS_HEADER_SIZE,
406 		str->data_len-LDNS_HEADER_SIZE);
407 	if(!dname_len) {
408 		s = strdup("query_dname_not_valid");
409 		if(!s)
410 			fatal_exit("strdup failed: out of memory");
411 		return s;
412 	}
413 	(void)sldns_wire2str_dname_buf(str->data+LDNS_HEADER_SIZE, dname_len,
414 		dname, sizeof(dname));
415 	tp = sldns_wirerr_get_type(str->data+LDNS_HEADER_SIZE,
416 		str->data_len-LDNS_HEADER_SIZE, dname_len);
417 	(void)sldns_wire2str_type_buf(tp, tpstr, sizeof(tpstr));
418 	snprintf(result, sizeof(result), "%s %s", dname, tpstr);
419 	s = strdup(result);
420 	if(!s)
421 		fatal_exit("strdup failed: out of memory");
422 	return s;
423 }
424 
425 /** create query stream list */
426 static struct doq_client_stream_list*
427 stream_list_create(void)
428 {
429 	struct doq_client_stream_list* list = calloc(1, sizeof(*list));
430 	if(!list)
431 		fatal_exit("calloc failed: out of memory");
432 	return list;
433 }
434 
435 /** free the query stream list */
436 static void
437 stream_list_free(struct doq_client_stream_list* list)
438 {
439 	struct doq_client_stream* str;
440 	if(!list)
441 		return;
442 	str = list->first;
443 	while(str) {
444 		struct doq_client_stream* next = str->next;
445 		client_stream_free(str);
446 		str = next;
447 	}
448 	free(list);
449 }
450 
451 /** append item to list */
452 static void
453 stream_list_append(struct doq_client_stream_list* list,
454 	struct doq_client_stream* str)
455 {
456 	if(list->last) {
457 		str->prev = list->last;
458 		list->last->next = str;
459 	} else {
460 		str->prev = NULL;
461 		list->first = str;
462 	}
463 	str->next = NULL;
464 	list->last = str;
465 }
466 
467 /** delete the item from the list */
468 static void
469 stream_list_delete(struct doq_client_stream_list* list,
470 	struct doq_client_stream* str)
471 {
472 	if(str->next) {
473 		str->next->prev = str->prev;
474 	} else {
475 		list->last = str->prev;
476 	}
477 	if(str->prev) {
478 		str->prev->next = str->next;
479 	} else {
480 		list->first = str->next;
481 	}
482 	str->prev = NULL;
483 	str->next = NULL;
484 }
485 
486 /** move the item from list1 to list2 */
487 static void
488 stream_list_move(struct doq_client_stream* str,
489 	struct doq_client_stream_list* list1,
490 	struct doq_client_stream_list* list2)
491 {
492 	stream_list_delete(list1, str);
493 	stream_list_append(list2, str);
494 }
495 
496 /** allocate stream data buffer, then answer length is complete */
497 static void
498 client_stream_datalen_complete(struct doq_client_stream* str)
499 {
500 	verbose(1, "answer length %d", (int)ntohs(str->answer_len));
501 	str->answer = sldns_buffer_new(ntohs(str->answer_len));
502 	if(!str->answer)
503 		fatal_exit("sldns_buffer_new failed: out of memory");
504 	sldns_buffer_set_limit(str->answer, ntohs(str->answer_len));
505 }
506 
507 /** print the answer rrs */
508 static void
509 print_answer_rrs(uint8_t* pkt, size_t pktlen)
510 {
511 	char buf[65535];
512 	char* str;
513 	size_t str_len;
514 	int i, qdcount, ancount;
515 	uint8_t* data = pkt;
516 	size_t data_len = pktlen;
517 	int comprloop = 0;
518 	if(data_len < LDNS_HEADER_SIZE)
519 		return;
520 	qdcount = LDNS_QDCOUNT(data);
521 	ancount = LDNS_ANCOUNT(data);
522 	data += LDNS_HEADER_SIZE;
523 	data_len -= LDNS_HEADER_SIZE;
524 
525 	for(i=0; i<qdcount; i++) {
526 		str = buf;
527 		str_len = sizeof(buf);
528 		(void)sldns_wire2str_rrquestion_scan(&data, &data_len,
529 			&str, &str_len, pkt, pktlen, &comprloop);
530 	}
531 	for(i=0; i<ancount; i++) {
532 		str = buf;
533 		str_len = sizeof(buf);
534 		(void)sldns_wire2str_rr_scan(&data, &data_len, &str, &str_len,
535 			pkt, pktlen, &comprloop);
536 		/* terminate string */
537 		if(str_len == 0)
538 			buf[sizeof(buf)-1] = 0;
539 		else	*str = 0;
540 		printf("%s", buf);
541 	}
542 }
543 
544 /** short output of answer, short error or rcode or answer section RRs. */
545 static void
546 client_stream_print_short(struct doq_client_stream* str)
547 {
548 	int rcode, ancount;
549 	if(str->query_has_error) {
550 		char* logs = client_stream_string(str);
551 		printf("%s has error, there is no answer\n", logs);
552 		free(logs);
553 		return;
554 	}
555 	if(sldns_buffer_limit(str->answer) < LDNS_HEADER_SIZE) {
556 		char* logs = client_stream_string(str);
557 		printf("%s received short packet, smaller than header\n",
558 			logs);
559 		free(logs);
560 		return;
561 	}
562 	rcode = LDNS_RCODE_WIRE(sldns_buffer_begin(str->answer));
563 	if(rcode != 0) {
564 		char* logs = client_stream_string(str);
565 		char rc[16];
566 		(void)sldns_wire2str_rcode_buf(rcode, rc, sizeof(rc));
567 		printf("%s rcode %s\n", logs, rc);
568 		free(logs);
569 		return;
570 	}
571 	ancount = LDNS_ANCOUNT(sldns_buffer_begin(str->answer));
572 	if(ancount == 0) {
573 		char* logs = client_stream_string(str);
574 		printf("%s nodata answer\n", logs);
575 		free(logs);
576 		return;
577 	}
578 	print_answer_rrs(sldns_buffer_begin(str->answer),
579 		sldns_buffer_limit(str->answer));
580 }
581 
582 /** print the stream output answer */
583 static void
584 client_stream_print_long(struct doq_client_data* data,
585 	struct doq_client_stream* str)
586 {
587 	char* s;
588 	if(str->query_has_error) {
589 		char* logs = client_stream_string(str);
590 		printf("%s has error, there is no answer\n", logs);
591 		free(logs);
592 		return;
593 	}
594 	s = sldns_wire2str_pkt(sldns_buffer_begin(str->answer),
595 		sldns_buffer_limit(str->answer));
596 	printf("%s", (s?s:";sldns_wire2str_pkt failed\n"));
597 	printf(";; SERVER: %s %d\n", data->svr, data->port);
598 	free(s);
599 }
600 
601 /** the stream has completed the data */
602 static void
603 client_stream_data_complete(struct doq_client_stream* str)
604 {
605 	verbose(1, "received all answer content");
606 	if(verbosity > 0) {
607 		char* logs = client_stream_string(str);
608 		char* s;
609 		log_buf(1, "received answer", str->answer);
610 		s = sldns_wire2str_pkt(sldns_buffer_begin(str->answer),
611 			sldns_buffer_limit(str->answer));
612 		if(!s) verbose(1, "could not sldns_wire2str_pkt");
613 		else verbose(1, "query %s received:\n%s", logs, s);
614 		free(s);
615 		free(logs);
616 	}
617 	str->answer_is_complete = 1;
618 }
619 
620 /** the stream has completed but with an error */
621 static void
622 client_stream_answer_error(struct doq_client_stream* str)
623 {
624 	if(verbosity > 0) {
625 		char* logs = client_stream_string(str);
626 		if(str->answer)
627 			verbose(1, "query %s has an error. received %d/%d bytes.",
628 				logs, (int)sldns_buffer_position(str->answer),
629 				(int)sldns_buffer_limit(str->answer));
630 		else
631 			verbose(1, "query %s has an error. received no data.",
632 				logs);
633 		free(logs);
634 	}
635 	str->query_has_error = 1;
636 }
637 
638 /** receive data for a stream */
639 static void
640 client_stream_recv_data(struct doq_client_stream* str, const uint8_t* data,
641 	size_t datalen)
642 {
643 	int got_data = 0;
644 	/* read the tcplength uint16_t at the start of the DNS message */
645 	if(str->nread < 2) {
646 		size_t to_move = datalen;
647 		if(datalen > 2-str->nread)
648 			to_move = 2-str->nread;
649 		memmove(((uint8_t*)&str->answer_len)+str->nread, data,
650 			to_move);
651 		str->nread += to_move;
652 		data += to_move;
653 		datalen -= to_move;
654 		if(str->nread == 2) {
655 			/* we can allocate the data buffer */
656 			client_stream_datalen_complete(str);
657 		}
658 	}
659 	/* if we have data bytes */
660 	if(datalen > 0) {
661 		size_t to_write = datalen;
662 		if(datalen > sldns_buffer_remaining(str->answer))
663 			to_write = sldns_buffer_remaining(str->answer);
664 		if(to_write > 0) {
665 			sldns_buffer_write(str->answer, data, to_write);
666 			str->nread += to_write;
667 			data += to_write;
668 			datalen -= to_write;
669 			got_data = 1;
670 		}
671 	}
672 	/* extra received bytes after end? */
673 	if(datalen > 0) {
674 		verbose(1, "extra bytes after end of DNS length");
675 		if(verbosity > 0)
676 			log_hex("extradata", (void*)data, datalen);
677 	}
678 	/* are we done with it? */
679 	if(got_data && str->nread >= (size_t)(ntohs(str->answer_len))+2) {
680 		client_stream_data_complete(str);
681 	}
682 }
683 
684 /** receive FIN from remote end on client stream, no more data to be
685  * received on the stream. */
686 static void
687 client_stream_recv_fin(struct doq_client_data* data,
688 	struct doq_client_stream* str, int is_fin)
689 {
690 	if(verbosity > 0) {
691 		char* logs = client_stream_string(str);
692 		if(is_fin)
693 			verbose(1, "query %s: received FIN from remote", logs);
694 		else
695 			verbose(1, "query %s: stream reset from remote", logs);
696 		free(logs);
697 	}
698 	if(str->write_is_done)
699 		stream_list_move(str, data->query_list_receive,
700 			data->query_list_stop);
701 	else
702 		stream_list_move(str, data->query_list_send,
703 			data->query_list_stop);
704 	if(!str->answer_is_complete) {
705 		client_stream_answer_error(str);
706 	}
707 	str->query_is_done = 1;
708 	if(data->quiet)
709 		client_stream_print_short(str);
710 	else client_stream_print_long(data, str);
711 	if(data->query_list_send->first==NULL &&
712 		data->query_list_receive->first==NULL)
713 		disconnect(data);
714 }
715 
716 /** fill a buffer with random data */
717 static void fill_rand(struct ub_randstate* rnd, uint8_t* buf, size_t len)
718 {
719 	if(RAND_bytes(buf, len) != 1) {
720 		size_t i;
721 		for(i=0; i<len; i++)
722 			buf[i] = ub_random(rnd)&0xff;
723 	}
724 }
725 
726 /** create the static secret */
727 static void generate_static_secret(struct doq_client_data* data, size_t len)
728 {
729 	data->static_secret_data = malloc(len);
730 	if(!data->static_secret_data)
731 		fatal_exit("malloc failed: out of memory");
732 	data->static_secret_size = len;
733 	fill_rand(data->rnd, data->static_secret_data, len);
734 }
735 
736 /** fill cid structure with random data */
737 static void cid_randfill(struct ngtcp2_cid* cid, size_t datalen,
738 	struct ub_randstate* rnd)
739 {
740 	uint8_t buf[32];
741 	if(datalen > sizeof(buf))
742 		datalen = sizeof(buf);
743 	fill_rand(rnd, buf, datalen);
744 	ngtcp2_cid_init(cid, buf, datalen);
745 }
746 
747 /** send buf on the client stream */
748 static int
749 client_bidi_stream(struct doq_client_data* data, int64_t* ret_stream_id,
750 	void* stream_user_data)
751 {
752 	int64_t stream_id;
753 	int rv;
754 
755 	/* open new bidirectional stream */
756 	rv = ngtcp2_conn_open_bidi_stream(data->conn, &stream_id,
757 		stream_user_data);
758 	if(rv != 0) {
759 		if(rv == NGTCP2_ERR_STREAM_ID_BLOCKED) {
760 			/* no bidi stream count for this new stream */
761 			return 0;
762 		}
763 		fatal_exit("could not ngtcp2_conn_open_bidi_stream: %s",
764 			ngtcp2_strerror(rv));
765 	}
766 	*ret_stream_id = stream_id;
767 	return 1;
768 }
769 
770 /** See if we can start query streams, by creating bidirectional streams
771  * on the QUIC transport for them. */
772 static void
773 query_streams_start(struct doq_client_data* data)
774 {
775 	while(data->query_list_start->first) {
776 		struct doq_client_stream* str = data->query_list_start->first;
777 		int64_t stream_id = 0;
778 		if(!client_bidi_stream(data, &stream_id, str)) {
779 			/* no more bidi streams allowed */
780 			break;
781 		}
782 		if(verbosity > 0) {
783 			char* logs = client_stream_string(str);
784 			verbose(1, "query %s start on bidi stream id %lld",
785 				logs, (long long int)stream_id);
786 			free(logs);
787 		}
788 		/* setup the stream to start */
789 		client_stream_start_setup(str, stream_id);
790 		/* move the query entry to the send list to write it */
791 		stream_list_move(str, data->query_list_start,
792 			data->query_list_send);
793 	}
794 }
795 
796 /** the rand callback routine from ngtcp2 */
797 static void rand_cb(uint8_t* dest, size_t destlen,
798 	const ngtcp2_rand_ctx* rand_ctx)
799 {
800 	struct ub_randstate* rnd = (struct ub_randstate*)
801 		rand_ctx->native_handle;
802 	fill_rand(rnd, dest, destlen);
803 }
804 
805 /** the get_new_connection_id callback routine from ngtcp2 */
806 static int get_new_connection_id_cb(struct ngtcp2_conn* ATTR_UNUSED(conn),
807 	struct ngtcp2_cid* cid, uint8_t* token, size_t cidlen, void* user_data)
808 {
809 	struct doq_client_data* data = (struct doq_client_data*)user_data;
810 	cid_randfill(cid, cidlen, data->rnd);
811 	if(ngtcp2_crypto_generate_stateless_reset_token(token,
812 		data->static_secret_data, data->static_secret_size, cid) != 0)
813 		return NGTCP2_ERR_CALLBACK_FAILURE;
814 	return 0;
815 }
816 
817 /** handle that early data is rejected */
818 static void
819 early_data_is_rejected(struct doq_client_data* data)
820 {
821 	int rv;
822 	verbose(1, "early data was rejected by the server");
823 #ifdef HAVE_NGTCP2_CONN_TLS_EARLY_DATA_REJECTED
824 	rv = ngtcp2_conn_tls_early_data_rejected(data->conn);
825 #else
826 	rv = ngtcp2_conn_early_data_rejected(data->conn);
827 #endif
828 	if(rv != 0) {
829 		log_err("ngtcp2_conn_early_data_rejected failed: %s",
830 			ngtcp2_strerror(rv));
831 		return;
832 	}
833 	/* move the streams back to the start state */
834 	while(data->query_list_send->first) {
835 		struct doq_client_stream* str = data->query_list_send->first;
836 		/* move it back to the start list */
837 		stream_list_move(str, data->query_list_send,
838 			data->query_list_start);
839 		str->has_stream = 0;
840 		/* remove stream id */
841 		str->stream_id = 0;
842 		/* initialise other members, in case they are altered,
843 		 * but unlikely, because early streams are rejected. */
844 		str->nwrite = 0;
845 		str->nread = 0;
846 		str->answer_len = 0;
847 		str->query_is_done = 0;
848 		str->answer_is_complete = 0;
849 		str->query_has_error = 0;
850 		if(str->answer) {
851 			sldns_buffer_free(str->answer);
852 			str->answer = NULL;
853 		}
854 	}
855 }
856 
857 /** the handshake completed callback from ngtcp2 */
858 static int
859 handshake_completed(ngtcp2_conn* ATTR_UNUSED(conn), void* user_data)
860 {
861 	struct doq_client_data* data = (struct doq_client_data*)user_data;
862 	verbose(1, "handshake_completed callback");
863 	verbose(1, "ngtcp2_conn_get_max_data_left is %d",
864 		(int)ngtcp2_conn_get_max_data_left(data->conn));
865 #ifdef HAVE_NGTCP2_CONN_GET_MAX_LOCAL_STREAMS_UNI
866 	verbose(1, "ngtcp2_conn_get_max_local_streams_uni is %d",
867 		(int)ngtcp2_conn_get_max_local_streams_uni(data->conn));
868 #endif
869 	verbose(1, "ngtcp2_conn_get_streams_uni_left is %d",
870 		(int)ngtcp2_conn_get_streams_uni_left(data->conn));
871 	verbose(1, "ngtcp2_conn_get_streams_bidi_left is %d",
872 		(int)ngtcp2_conn_get_streams_bidi_left(data->conn));
873 	verbose(1, "negotiated cipher name is %s",
874 		SSL_get_cipher_name(data->ssl));
875 	if(verbosity > 0) {
876 		const unsigned char* alpn = NULL;
877 		unsigned int alpnlen = 0;
878 		char alpnstr[128];
879 		SSL_get0_alpn_selected(data->ssl, &alpn, &alpnlen);
880 		if(alpnlen > sizeof(alpnstr)-1)
881 			alpnlen = sizeof(alpnstr)-1;
882 		memmove(alpnstr, alpn, alpnlen);
883 		alpnstr[alpnlen]=0;
884 		verbose(1, "negotiated ALPN is '%s'", alpnstr);
885 	}
886 	/* The SSL_get_early_data_status call works after the handshake
887 	 * completes. */
888 	if(data->early_data_enabled) {
889 		if(SSL_get_early_data_status(data->ssl) !=
890 			SSL_EARLY_DATA_ACCEPTED) {
891 			early_data_is_rejected(data);
892 		} else {
893 			verbose(1, "early data was accepted by the server");
894 		}
895 	}
896 #ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
897 	if(data->transport_file) {
898 		early_data_write_transport(data);
899 	}
900 #endif
901 	return 0;
902 }
903 
904 /** the extend_max_local_streams_bidi callback from ngtcp2 */
905 static int
906 extend_max_local_streams_bidi(ngtcp2_conn* ATTR_UNUSED(conn),
907 	uint64_t max_streams, void* user_data)
908 {
909 	struct doq_client_data* data = (struct doq_client_data*)user_data;
910 	verbose(1, "extend_max_local_streams_bidi callback, %d max_streams",
911 		(int)max_streams);
912 	verbose(1, "ngtcp2_conn_get_max_data_left is %d",
913 		(int)ngtcp2_conn_get_max_data_left(data->conn));
914 #ifdef HAVE_NGTCP2_CONN_GET_MAX_LOCAL_STREAMS_UNI
915 	verbose(1, "ngtcp2_conn_get_max_local_streams_uni is %d",
916 		(int)ngtcp2_conn_get_max_local_streams_uni(data->conn));
917 #endif
918 	verbose(1, "ngtcp2_conn_get_streams_uni_left is %d",
919 		(int)ngtcp2_conn_get_streams_uni_left(data->conn));
920 	verbose(1, "ngtcp2_conn_get_streams_bidi_left is %d",
921 		(int)ngtcp2_conn_get_streams_bidi_left(data->conn));
922 	query_streams_start(data);
923 	return 0;
924 }
925 
926 /** the recv_stream_data callback from ngtcp2 */
927 static int
928 recv_stream_data(ngtcp2_conn* ATTR_UNUSED(conn), uint32_t flags,
929 	int64_t stream_id, uint64_t offset, const uint8_t* data,
930 	size_t datalen, void* user_data, void* stream_user_data)
931 {
932 	struct doq_client_data* doqdata = (struct doq_client_data*)user_data;
933 	struct doq_client_stream* str = (struct doq_client_stream*)
934 		stream_user_data;
935 	verbose(1, "recv_stream_data stream %d offset %d datalen %d%s%s",
936 		(int)stream_id, (int)offset, (int)datalen,
937 		((flags&NGTCP2_STREAM_DATA_FLAG_FIN)!=0?" FIN":""),
938 #ifdef NGTCP2_STREAM_DATA_FLAG_0RTT
939 		((flags&NGTCP2_STREAM_DATA_FLAG_0RTT)!=0?" 0RTT":"")
940 #else
941 		((flags&NGTCP2_STREAM_DATA_FLAG_EARLY)!=0?" EARLY":"")
942 #endif
943 		);
944 	if(verbosity > 0)
945 		log_hex("data", (void*)data, datalen);
946 	if(verbosity > 0) {
947 		char* logs = client_stream_string(str);
948 		verbose(1, "the stream_user_data is %s stream id %d, nread %d",
949 			logs, (int)str->stream_id, (int)str->nread);
950 		free(logs);
951 	}
952 
953 	/* append the data, if there is data */
954 	if(datalen > 0) {
955 		client_stream_recv_data(str, data, datalen);
956 	}
957 	if((flags&NGTCP2_STREAM_DATA_FLAG_FIN)!=0) {
958 		client_stream_recv_fin(doqdata, str, 1);
959 	}
960 	ngtcp2_conn_extend_max_stream_offset(doqdata->conn, stream_id, datalen);
961 	ngtcp2_conn_extend_max_offset(doqdata->conn, datalen);
962 	return 0;
963 }
964 
965 /** the stream reset callback from ngtcp2 */
966 static int
967 stream_reset(ngtcp2_conn* ATTR_UNUSED(conn), int64_t stream_id,
968 	uint64_t final_size, uint64_t app_error_code, void* user_data,
969 	void* stream_user_data)
970 {
971 	struct doq_client_data* doqdata = (struct doq_client_data*)user_data;
972 	struct doq_client_stream* str = (struct doq_client_stream*)
973 		stream_user_data;
974 	verbose(1, "stream reset for stream %d final size %d app error code %d",
975 		(int)stream_id, (int)final_size, (int)app_error_code);
976 	client_stream_recv_fin(doqdata, str, 0);
977 	return 0;
978 }
979 
980 /** copy sockaddr into ngtcp2 addr */
981 static void
982 copy_ngaddr(struct ngtcp2_addr* ngaddr, struct sockaddr_storage* addr,
983 	socklen_t addrlen)
984 {
985 	if(addr_is_ip6(addr, addrlen)) {
986 #if defined(NGTCP2_USE_GENERIC_SOCKADDR) || defined(NGTCP2_USE_GENERIC_IPV6_SOCKADDR)
987 		struct sockaddr_in* i6 = (struct sockaddr_in6*)addr;
988 		struct ngtcp2_sockaddr_in6 a6;
989 		ngaddr->addr = calloc(1, sizeof(a6));
990 		if(!ngaddr->addr) fatal_exit("calloc failed: out of memory");
991 		ngaddr->addrlen = sizeof(a6);
992 		memset(&a6, 0, sizeof(a6));
993 		a6.sin6_family = i6->sin6_family;
994 		a6.sin6_port = i6->sin6_port;
995 		a6.sin6_flowinfo = i6->sin6_flowinfo;
996 		memmove(&a6.sin6_addr, i6->sin6_addr, sizeof(a6.sin6_addr);
997 		a6.sin6_scope_id = i6->sin6_scope_id;
998 		memmove(ngaddr->addr, &a6, sizeof(a6));
999 #else
1000 		ngaddr->addr = (ngtcp2_sockaddr*)addr;
1001 		ngaddr->addrlen = addrlen;
1002 #endif
1003 	} else {
1004 #ifdef NGTCP2_USE_GENERIC_SOCKADDR
1005 		struct sockaddr_in* i4 = (struct sockaddr_in*)addr;
1006 		struct ngtcp2_sockaddr_in a4;
1007 		ngaddr->addr = calloc(1, sizeof(a4));
1008 		if(!ngaddr->addr) fatal_exit("calloc failed: out of memory");
1009 		ngaddr->addrlen = sizeof(a4);
1010 		memset(&a4, 0, sizeof(a4));
1011 		a4.sin_family = i4->sin_family;
1012 		a4.sin_port = i4->sin_port;
1013 		memmove(&a4.sin_addr, i4->sin_addr, sizeof(a4.sin_addr);
1014 		memmove(ngaddr->addr, &a4, sizeof(a4));
1015 #else
1016 		ngaddr->addr = (ngtcp2_sockaddr*)addr;
1017 		ngaddr->addrlen = addrlen;
1018 #endif
1019 	}
1020 }
1021 
1022 /** debug log printf for ngtcp2 connections */
1023 static void log_printf_for_doq(void* ATTR_UNUSED(user_data),
1024 	const char* fmt, ...)
1025 {
1026 	va_list ap;
1027 	va_start(ap, fmt);
1028 	fprintf(stderr, "libngtcp2: ");
1029 	vfprintf(stderr, fmt, ap);
1030 	va_end(ap);
1031 	fprintf(stderr, "\n");
1032 }
1033 
1034 /** get a timestamp in nanoseconds */
1035 static ngtcp2_tstamp get_timestamp_nanosec(void)
1036 {
1037 #ifdef CLOCK_REALTIME
1038 	struct timespec tp;
1039 	memset(&tp, 0, sizeof(tp));
1040 #ifdef CLOCK_MONOTONIC
1041 	if(clock_gettime(CLOCK_MONOTONIC, &tp) == -1) {
1042 #endif
1043 		if(clock_gettime(CLOCK_REALTIME, &tp) == -1) {
1044 			log_err("clock_gettime failed: %s", strerror(errno));
1045 		}
1046 #ifdef CLOCK_MONOTONIC
1047 	}
1048 #endif
1049 	return ((uint64_t)tp.tv_sec)*((uint64_t)1000000000) +
1050 		((uint64_t)tp.tv_nsec);
1051 #else
1052 	struct timeval tv;
1053 	if(gettimeofday(&tv, NULL) < 0) {
1054 		log_err("gettimeofday failed: %s", strerror(errno));
1055 	}
1056 	return ((uint64_t)tv.tv_sec)*((uint64_t)1000000000) +
1057 		((uint64_t)tv.tv_usec)*((uint64_t)1000);
1058 #endif /* CLOCK_REALTIME */
1059 }
1060 
1061 /** create ngtcp2 client connection and set up. */
1062 static struct ngtcp2_conn* conn_client_setup(struct doq_client_data* data)
1063 {
1064 	struct ngtcp2_conn* conn = NULL;
1065 	int rv;
1066 	struct ngtcp2_cid dcid, scid;
1067 	struct ngtcp2_path path;
1068 	uint32_t client_chosen_version = NGTCP2_PROTO_VER_V1;
1069 	struct ngtcp2_callbacks cbs;
1070 	struct ngtcp2_settings settings;
1071 	struct ngtcp2_transport_params params;
1072 
1073 	memset(&cbs, 0, sizeof(cbs));
1074 	memset(&settings, 0, sizeof(settings));
1075 	memset(&params, 0, sizeof(params));
1076 	memset(&dcid, 0, sizeof(dcid));
1077 	memset(&scid, 0, sizeof(scid));
1078 	memset(&path, 0, sizeof(path));
1079 
1080 	data->quic_version = client_chosen_version;
1081 	ngtcp2_settings_default(&settings);
1082 	if(str_is_ip6(data->svr)) {
1083 #ifdef HAVE_STRUCT_NGTCP2_SETTINGS_MAX_TX_UDP_PAYLOAD_SIZE
1084 		settings.max_tx_udp_payload_size = 1232;
1085 #else
1086 		settings.max_udp_payload_size = 1232;
1087 #endif
1088 	}
1089 	settings.rand_ctx.native_handle = data->rnd;
1090 	if(verbosity > 0) {
1091 		/* make debug logs */
1092 		settings.log_printf = log_printf_for_doq;
1093 	}
1094 	settings.initial_ts = get_timestamp_nanosec();
1095 	ngtcp2_transport_params_default(&params);
1096 	params.initial_max_stream_data_bidi_local = 256*1024;
1097 	params.initial_max_stream_data_bidi_remote = 256*1024;
1098 	params.initial_max_stream_data_uni = 256*1024;
1099 	params.initial_max_data = 1024*1024;
1100 	params.initial_max_streams_bidi = 0;
1101 	params.initial_max_streams_uni = 100;
1102 	params.max_idle_timeout = 30*NGTCP2_SECONDS;
1103 	params.active_connection_id_limit = 7;
1104 	cid_randfill(&dcid, 16, data->rnd);
1105 	cid_randfill(&scid, 16, data->rnd);
1106 	cbs.client_initial = ngtcp2_crypto_client_initial_cb;
1107 	cbs.recv_crypto_data = ngtcp2_crypto_recv_crypto_data_cb;
1108 	cbs.encrypt = ngtcp2_crypto_encrypt_cb;
1109 	cbs.decrypt = ngtcp2_crypto_decrypt_cb;
1110 	cbs.hp_mask = ngtcp2_crypto_hp_mask_cb;
1111 	cbs.recv_retry = ngtcp2_crypto_recv_retry_cb;
1112 	cbs.update_key = ngtcp2_crypto_update_key_cb;
1113 	cbs.delete_crypto_aead_ctx = ngtcp2_crypto_delete_crypto_aead_ctx_cb;
1114 	cbs.delete_crypto_cipher_ctx =
1115 		ngtcp2_crypto_delete_crypto_cipher_ctx_cb;
1116 	cbs.get_path_challenge_data = ngtcp2_crypto_get_path_challenge_data_cb;
1117 	cbs.version_negotiation = ngtcp2_crypto_version_negotiation_cb;
1118 	cbs.get_new_connection_id = get_new_connection_id_cb;
1119 	cbs.handshake_completed = handshake_completed;
1120 	cbs.extend_max_local_streams_bidi = extend_max_local_streams_bidi;
1121 	cbs.rand = rand_cb;
1122 	cbs.recv_stream_data = recv_stream_data;
1123 	cbs.stream_reset = stream_reset;
1124 	copy_ngaddr(&path.local, &data->local_addr, data->local_addr_len);
1125 	copy_ngaddr(&path.remote, &data->dest_addr, data->dest_addr_len);
1126 
1127 	rv = ngtcp2_conn_client_new(&conn, &dcid, &scid, &path,
1128 		client_chosen_version, &cbs, &settings, &params,
1129 		NULL, /* ngtcp2_mem allocator, use default */
1130 		data /* callback argument */);
1131 	if(!conn) fatal_exit("could not ngtcp2_conn_client_new: %s",
1132 		ngtcp2_strerror(rv));
1133 	data->cc_algo = settings.cc_algo;
1134 	return conn;
1135 }
1136 
1137 #ifndef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
1138 /** write the transport file */
1139 static void
1140 transport_file_write(const char* file, struct ngtcp2_transport_params* params)
1141 {
1142 	FILE* out;
1143 	out = fopen(file, "w");
1144 	if(!out) {
1145 		perror(file);
1146 		return;
1147 	}
1148 	fprintf(out, "initial_max_streams_bidi=%u\n",
1149 		(unsigned)params->initial_max_streams_bidi);
1150 	fprintf(out, "initial_max_streams_uni=%u\n",
1151 		(unsigned)params->initial_max_streams_uni);
1152 	fprintf(out, "initial_max_stream_data_bidi_local=%u\n",
1153 		(unsigned)params->initial_max_stream_data_bidi_local);
1154 	fprintf(out, "initial_max_stream_data_bidi_remote=%u\n",
1155 		(unsigned)params->initial_max_stream_data_bidi_remote);
1156 	fprintf(out, "initial_max_stream_data_uni=%u\n",
1157 		(unsigned)params->initial_max_stream_data_uni);
1158 	fprintf(out, "initial_max_data=%u\n",
1159 		(unsigned)params->initial_max_data);
1160 	fprintf(out, "active_connection_id_limit=%u\n",
1161 		(unsigned)params->active_connection_id_limit);
1162 	fprintf(out, "max_datagram_frame_size=%u\n",
1163 		(unsigned)params->max_datagram_frame_size);
1164 	if(ferror(out)) {
1165 		verbose(1, "There was an error writing %s: %s",
1166 			file, strerror(errno));
1167 		fclose(out);
1168 		return;
1169 	}
1170 	fclose(out);
1171 }
1172 #endif /* HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS */
1173 
1174 /** fetch and write the transport file */
1175 static void
1176 early_data_write_transport(struct doq_client_data* data)
1177 {
1178 #ifdef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
1179 	FILE* out;
1180 	uint8_t buf[1024];
1181 	ngtcp2_ssize len = ngtcp2_conn_encode_0rtt_transport_params(data->conn,
1182 		buf, sizeof(buf));
1183 	if(len < 0) {
1184 		log_err("ngtcp2_conn_encode_0rtt_transport_params failed: %s",
1185 			ngtcp2_strerror(len));
1186 		return;
1187 	}
1188 	out = fopen(data->transport_file, "w");
1189 	if(!out) {
1190 		perror(data->transport_file);
1191 		return;
1192 	}
1193 	if(fwrite(buf, 1, len, out) != (size_t)len) {
1194 		log_err("fwrite %s failed: %s", data->transport_file,
1195 			strerror(errno));
1196 	}
1197 	if(ferror(out)) {
1198 		verbose(1, "There was an error writing %s: %s",
1199 			data->transport_file, strerror(errno));
1200 	}
1201 	fclose(out);
1202 #else
1203 	struct ngtcp2_transport_params params;
1204 	memset(&params, 0, sizeof(params));
1205 	ngtcp2_conn_get_remote_transport_params(data->conn, &params);
1206 	transport_file_write(data->transport_file, &params);
1207 #endif
1208 }
1209 
1210 #ifndef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
1211 /** applicatation rx key callback, this is where the rx key is set,
1212  * and streams can be opened, like http3 unidirectional streams, like
1213  * the http3 control and http3 qpack encode and decoder streams. */
1214 static int
1215 application_rx_key_cb(struct doq_client_data* data)
1216 {
1217 	verbose(1, "application_rx_key_cb callback");
1218 	verbose(1, "ngtcp2_conn_get_max_data_left is %d",
1219 		(int)ngtcp2_conn_get_max_data_left(data->conn));
1220 #ifdef HAVE_NGTCP2_CONN_GET_MAX_LOCAL_STREAMS_UNI
1221 	verbose(1, "ngtcp2_conn_get_max_local_streams_uni is %d",
1222 		(int)ngtcp2_conn_get_max_local_streams_uni(data->conn));
1223 #endif
1224 	verbose(1, "ngtcp2_conn_get_streams_uni_left is %d",
1225 		(int)ngtcp2_conn_get_streams_uni_left(data->conn));
1226 	verbose(1, "ngtcp2_conn_get_streams_bidi_left is %d",
1227 		(int)ngtcp2_conn_get_streams_bidi_left(data->conn));
1228 	if(data->transport_file) {
1229 		early_data_write_transport(data);
1230 	}
1231 	return 1;
1232 }
1233 
1234 /** quic_method set_encryption_secrets function */
1235 static int
1236 set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
1237 	const uint8_t *read_secret, const uint8_t *write_secret,
1238 	size_t secret_len)
1239 {
1240 	struct doq_client_data* data = get_app_data(ssl);
1241 #ifdef HAVE_NGTCP2_ENCRYPTION_LEVEL
1242 	ngtcp2_encryption_level
1243 #else
1244 	ngtcp2_crypto_level
1245 #endif
1246 		level =
1247 #ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_FROM_OSSL_ENCRYPTION_LEVEL
1248 		ngtcp2_crypto_quictls_from_ossl_encryption_level(ossl_level);
1249 #else
1250 		ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level);
1251 #endif
1252 
1253 	if(read_secret) {
1254 		if(ngtcp2_crypto_derive_and_install_rx_key(data->conn, NULL,
1255 			NULL, NULL, level, read_secret, secret_len) != 0) {
1256 			log_err("ngtcp2_crypto_derive_and_install_rx_key failed");
1257 			return 0;
1258 		}
1259 		if(level == NGTCP2_CRYPTO_LEVEL_APPLICATION) {
1260 			if(!application_rx_key_cb(data))
1261 				return 0;
1262 		}
1263 	}
1264 
1265 	if(write_secret) {
1266 		if(ngtcp2_crypto_derive_and_install_tx_key(data->conn, NULL,
1267 			NULL, NULL, level, write_secret, secret_len) != 0) {
1268 			log_err("ngtcp2_crypto_derive_and_install_tx_key failed");
1269 			return 0;
1270 		}
1271 	}
1272 	return 1;
1273 }
1274 
1275 /** quic_method add_handshake_data function */
1276 static int
1277 add_handshake_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
1278 	const uint8_t *data, size_t len)
1279 {
1280 	struct doq_client_data* doqdata = get_app_data(ssl);
1281 #ifdef HAVE_NGTCP2_ENCRYPTION_LEVEL
1282 	ngtcp2_encryption_level
1283 #else
1284 	ngtcp2_crypto_level
1285 #endif
1286 		level =
1287 #ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_FROM_OSSL_ENCRYPTION_LEVEL
1288 		ngtcp2_crypto_quictls_from_ossl_encryption_level(ossl_level);
1289 #else
1290 		ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level);
1291 #endif
1292 	int rv;
1293 
1294 	rv = ngtcp2_conn_submit_crypto_data(doqdata->conn, level, data, len);
1295 	if(rv != 0) {
1296 		log_err("ngtcp2_conn_submit_crypto_data failed: %s",
1297 			ngtcp2_strerror(rv));
1298 		ngtcp2_conn_set_tls_error(doqdata->conn, rv);
1299 		return 0;
1300 	}
1301 	return 1;
1302 }
1303 
1304 /** quic_method flush_flight function */
1305 static int
1306 flush_flight(SSL* ATTR_UNUSED(ssl))
1307 {
1308 	return 1;
1309 }
1310 
1311 /** quic_method send_alert function */
1312 static int
1313 send_alert(SSL *ssl, enum ssl_encryption_level_t ATTR_UNUSED(level),
1314 	uint8_t alert)
1315 {
1316 	struct doq_client_data* data = get_app_data(ssl);
1317 	data->tls_alert = alert;
1318 	return 1;
1319 }
1320 #endif /* HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT */
1321 
1322 /** new session callback. We can write it to file for resumption later. */
1323 static int
1324 new_session_cb(SSL* ssl, SSL_SESSION* session)
1325 {
1326 	struct doq_client_data* data = get_app_data(ssl);
1327 	BIO *f;
1328 	log_assert(data->session_file);
1329 	verbose(1, "new session cb: the ssl session max_early_data_size is %u",
1330 		(unsigned)SSL_SESSION_get_max_early_data(session));
1331 	f = BIO_new_file(data->session_file, "w");
1332 	if(!f) {
1333 		log_err("Could not open %s: %s", data->session_file,
1334 			strerror(errno));
1335 		return 0;
1336 	}
1337 	PEM_write_bio_SSL_SESSION(f, session);
1338 	BIO_free(f);
1339 	verbose(1, "written tls session to %s", data->session_file);
1340 	return 0;
1341 }
1342 
1343 /** setup the TLS context */
1344 static SSL_CTX*
1345 ctx_client_setup(void)
1346 {
1347 	SSL_CTX* ctx = SSL_CTX_new(TLS_client_method());
1348 	if(!ctx) {
1349 		log_crypto_err("Could not SSL_CTX_new");
1350 		exit(1);
1351 	}
1352 	SSL_CTX_set_min_proto_version(ctx, TLS1_3_VERSION);
1353 	SSL_CTX_set_max_proto_version(ctx, TLS1_3_VERSION);
1354 	SSL_CTX_set_default_verify_paths(ctx);
1355 #ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
1356 	if(ngtcp2_crypto_quictls_configure_client_context(ctx) != 0) {
1357 		log_err("ngtcp2_crypto_quictls_configure_client_context failed");
1358 		exit(1);
1359 	}
1360 #else
1361 	memset(&quic_method, 0, sizeof(quic_method));
1362 	quic_method.set_encryption_secrets = &set_encryption_secrets;
1363 	quic_method.add_handshake_data = &add_handshake_data;
1364 	quic_method.flush_flight = &flush_flight;
1365 	quic_method.send_alert = &send_alert;
1366 	SSL_CTX_set_quic_method(ctx, &quic_method);
1367 #endif
1368 	return ctx;
1369 }
1370 
1371 
1372 /* setup the TLS object */
1373 static SSL*
1374 ssl_client_setup(struct doq_client_data* data)
1375 {
1376 	SSL* ssl = SSL_new(data->ctx);
1377 	if(!ssl) {
1378 		log_crypto_err("Could not SSL_new");
1379 		exit(1);
1380 	}
1381 	set_app_data(ssl, data);
1382 	SSL_set_connect_state(ssl);
1383 	if(!SSL_set_fd(ssl, data->fd)) {
1384 		log_crypto_err("Could not SSL_set_fd");
1385 		exit(1);
1386 	}
1387 	if((data->quic_version & 0xff000000) == 0xff000000) {
1388 		SSL_set_quic_use_legacy_codepoint(ssl, 1);
1389 	} else {
1390 		SSL_set_quic_use_legacy_codepoint(ssl, 0);
1391 	}
1392 	SSL_set_alpn_protos(ssl, (const unsigned char *)"\x03""doq", 4);
1393 	/* send the SNI host name */
1394 	SSL_set_tlsext_host_name(ssl, "localhost");
1395 	return ssl;
1396 }
1397 
1398 /** get packet ecn information */
1399 static uint32_t
1400 msghdr_get_ecn(struct msghdr* msg, int family)
1401 {
1402 #ifndef S_SPLINT_S
1403 	struct cmsghdr* cmsg;
1404 	if(family == AF_INET6) {
1405 		for(cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL;
1406 			cmsg = CMSG_NXTHDR(msg, cmsg)) {
1407 			if(cmsg->cmsg_level == IPPROTO_IPV6 &&
1408 				cmsg->cmsg_type == IPV6_TCLASS &&
1409 				cmsg->cmsg_len != 0) {
1410 				uint8_t* ecn = (uint8_t*)CMSG_DATA(cmsg);
1411 				return *ecn;
1412 			}
1413 		}
1414 		return 0;
1415 	}
1416 	for(cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL;
1417 		cmsg = CMSG_NXTHDR(msg, cmsg)) {
1418 		if(cmsg->cmsg_level == IPPROTO_IP &&
1419 			cmsg->cmsg_type == IP_TOS &&
1420 			cmsg->cmsg_len != 0) {
1421 			uint8_t* ecn = (uint8_t*)CMSG_DATA(cmsg);
1422 			return *ecn;
1423 		}
1424 	}
1425 	return 0;
1426 #endif /* S_SPLINT_S */
1427 }
1428 
1429 /** set the ecn on the transmission */
1430 static void
1431 set_ecn(int fd, int family, uint32_t ecn)
1432 {
1433 	unsigned int val = ecn;
1434 	if(family == AF_INET6) {
1435 		if(setsockopt(fd, IPPROTO_IPV6, IPV6_TCLASS, &val,
1436 			(socklen_t)sizeof(val)) == -1) {
1437 			log_err("setsockopt(.. IPV6_TCLASS ..): %s",
1438 				strerror(errno));
1439 		}
1440 		return;
1441 	}
1442 	if(setsockopt(fd, IPPROTO_IP, IP_TOS, &val,
1443 		(socklen_t)sizeof(val)) == -1) {
1444 		log_err("setsockopt(.. IP_TOS ..): %s",
1445 			strerror(errno));
1446 	}
1447 }
1448 
1449 /** send a packet */
1450 static int
1451 doq_client_send_pkt(struct doq_client_data* data, uint32_t ecn, uint8_t* buf,
1452 	size_t buf_len, int is_blocked_pkt, int* send_is_blocked)
1453 {
1454 	struct msghdr msg;
1455 	struct iovec iov[1];
1456 	ssize_t ret;
1457 	iov[0].iov_base = buf;
1458 	iov[0].iov_len = buf_len;
1459 	memset(&msg, 0, sizeof(msg));
1460 	msg.msg_name = (void*)&data->dest_addr;
1461 	msg.msg_namelen = data->dest_addr_len;
1462 	msg.msg_iov = iov;
1463 	msg.msg_iovlen = 1;
1464 	set_ecn(data->fd, data->dest_addr.ss_family, ecn);
1465 
1466 	for(;;) {
1467 		ret = sendmsg(data->fd, &msg, MSG_DONTWAIT);
1468 		if(ret == -1 && errno == EINTR)
1469 			continue;
1470 		break;
1471 	}
1472 	if(ret == -1) {
1473 		if(errno == EAGAIN) {
1474 			if(buf_len >
1475 				sldns_buffer_capacity(data->blocked_pkt))
1476 				return 0; /* Cannot store it, but the buffers
1477 				are equal length and large enough, so this
1478 				should not happen. */
1479 			data->have_blocked_pkt = 1;
1480 			if(send_is_blocked)
1481 				*send_is_blocked = 1;
1482 			/* If we already send the previously blocked packet,
1483 			 * no need to copy it, otherwise store the packet for
1484 			 * later. */
1485 			if(!is_blocked_pkt) {
1486 				data->blocked_pkt_pi.ecn = ecn;
1487 				sldns_buffer_clear(data->blocked_pkt);
1488 				sldns_buffer_write(data->blocked_pkt, buf,
1489 					buf_len);
1490 				sldns_buffer_flip(data->blocked_pkt);
1491 			}
1492 			return 0;
1493 		}
1494 		log_err("doq sendmsg: %s", strerror(errno));
1495 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1496 		ngtcp2_ccerr_set_application_error(&data->ccerr, -1, NULL, 0);
1497 #else
1498 		ngtcp2_connection_close_error_set_application_error(&data->last_error, -1, NULL, 0);
1499 #endif
1500 		return 0;
1501 	}
1502 	return 1;
1503 }
1504 
1505 /** change event write on fd to when we have data or when congested */
1506 static void
1507 event_change_write(struct doq_client_data* data, int do_write)
1508 {
1509 	ub_event_del(data->ev);
1510 	if(do_write) {
1511 		ub_event_add_bits(data->ev, UB_EV_WRITE);
1512 	} else {
1513 		ub_event_del_bits(data->ev, UB_EV_WRITE);
1514 	}
1515 	if(ub_event_add(data->ev, NULL) != 0) {
1516 		fatal_exit("could not ub_event_add");
1517 	}
1518 }
1519 
1520 /** write the connection close, with possible error */
1521 static void
1522 write_conn_close(struct doq_client_data* data)
1523 {
1524 	struct ngtcp2_path_storage ps;
1525 	struct ngtcp2_pkt_info pi;
1526 	ngtcp2_ssize ret;
1527 	if(!data->conn ||
1528 #ifdef HAVE_NGTCP2_CONN_IN_CLOSING_PERIOD
1529 		ngtcp2_conn_in_closing_period(data->conn) ||
1530 #else
1531 		ngtcp2_conn_is_in_closing_period(data->conn) ||
1532 #endif
1533 #ifdef HAVE_NGTCP2_CONN_IN_DRAINING_PERIOD
1534 		ngtcp2_conn_in_draining_period(data->conn)
1535 #else
1536 		ngtcp2_conn_is_in_draining_period(data->conn)
1537 #endif
1538 		)
1539 		return;
1540 	/* Drop blocked packet if there is one, the connection is being
1541 	 * closed. And thus no further data traffic. */
1542 	data->have_blocked_pkt = 0;
1543 	if(
1544 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1545 		data->ccerr.type == NGTCP2_CCERR_TYPE_IDLE_CLOSE
1546 #else
1547 		data->last_error.type ==
1548 		NGTCP2_CONNECTION_CLOSE_ERROR_CODE_TYPE_TRANSPORT_IDLE_CLOSE
1549 #endif
1550 		) {
1551 		/* do not call ngtcp2_conn_write_connection_close on the
1552 		 * connection because the ngtcp2_conn_handle_expiry call
1553 		 * has returned NGTCP2_ERR_IDLE_CLOSE. But continue to close
1554 		 * the connection. */
1555 		return;
1556 	}
1557 	verbose(1, "write connection close");
1558 	ngtcp2_path_storage_zero(&ps);
1559 	sldns_buffer_clear(data->pkt_buf);
1560 	ret = ngtcp2_conn_write_connection_close(
1561 		data->conn, &ps.path, &pi, sldns_buffer_begin(data->pkt_buf),
1562 		sldns_buffer_remaining(data->pkt_buf),
1563 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1564 		&data->ccerr
1565 #else
1566 		&data->last_error
1567 #endif
1568 		, get_timestamp_nanosec());
1569 	if(ret < 0) {
1570 		log_err("ngtcp2_conn_write_connection_close failed: %s",
1571 			ngtcp2_strerror(ret));
1572 		return;
1573 	}
1574 	verbose(1, "write connection close packet length %d", (int)ret);
1575 	if(ret == 0)
1576 		return;
1577 	doq_client_send_pkt(data, pi.ecn, sldns_buffer_begin(data->pkt_buf),
1578 		ret, 0, NULL);
1579 }
1580 
1581 /** disconnect we are done */
1582 static void
1583 disconnect(struct doq_client_data* data)
1584 {
1585 	verbose(1, "disconnect");
1586 	write_conn_close(data);
1587 	ub_event_base_loopexit(data->base);
1588 }
1589 
1590 /** the expire timer callback */
1591 void doq_client_timer_cb(int ATTR_UNUSED(fd),
1592 	short ATTR_UNUSED(bits), void* arg)
1593 {
1594 	struct doq_client_data* data = (struct doq_client_data*)arg;
1595 	ngtcp2_tstamp now = get_timestamp_nanosec();
1596 	int rv;
1597 
1598 	verbose(1, "doq expire_timer");
1599 	data->expire_timer_added = 0;
1600 	rv = ngtcp2_conn_handle_expiry(data->conn, now);
1601 	if(rv != 0) {
1602 		log_err("ngtcp2_conn_handle_expiry failed: %s",
1603 			ngtcp2_strerror(rv));
1604 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1605 		ngtcp2_ccerr_set_liberr(&data->ccerr, rv, NULL, 0);
1606 #else
1607 		ngtcp2_connection_close_error_set_transport_error_liberr(
1608 			&data->last_error, rv, NULL, 0);
1609 #endif
1610 		disconnect(data);
1611 		return;
1612 	}
1613 	update_timer(data);
1614 	on_write(data);
1615 }
1616 
1617 /** update the timers */
1618 static void
1619 update_timer(struct doq_client_data* data)
1620 {
1621 	ngtcp2_tstamp expiry = ngtcp2_conn_get_expiry(data->conn);
1622 	ngtcp2_tstamp now = get_timestamp_nanosec();
1623 	ngtcp2_tstamp t;
1624 	struct timeval tv;
1625 
1626 	if(expiry <= now) {
1627 		/* the timer has already expired, add with zero timeout */
1628 		t = 0;
1629 	} else {
1630 		t = expiry - now;
1631 	}
1632 
1633 	/* set the timer */
1634 	if(data->expire_timer_added) {
1635 		ub_timer_del(data->expire_timer);
1636 		data->expire_timer_added = 0;
1637 	}
1638 	memset(&tv, 0, sizeof(tv));
1639 	tv.tv_sec = t / NGTCP2_SECONDS;
1640 	tv.tv_usec = (t / NGTCP2_MICROSECONDS)%1000000;
1641 	verbose(1, "update_timer in %d.%6.6d secs", (int)tv.tv_sec,
1642 		(int)tv.tv_usec);
1643 	if(ub_timer_add(data->expire_timer, data->base,
1644 		&doq_client_timer_cb, data, &tv) != 0) {
1645 		log_err("timer_add failed: could not add expire timer");
1646 		return;
1647 	}
1648 	data->expire_timer_added = 1;
1649 }
1650 
1651 /** perform read operations on fd */
1652 static void
1653 on_read(struct doq_client_data* data)
1654 {
1655 	struct sockaddr_storage addr;
1656 	struct iovec iov[1];
1657 	struct msghdr msg;
1658 	union {
1659 		struct cmsghdr hdr;
1660 		char buf[256];
1661 	} ancil;
1662 	int i;
1663 	ssize_t rcv;
1664 	ngtcp2_pkt_info pi;
1665 	int rv;
1666 	struct ngtcp2_path path;
1667 
1668 	for(i=0; i<10; i++) {
1669 		msg.msg_name = &addr;
1670 		msg.msg_namelen = (socklen_t)sizeof(addr);
1671 		iov[0].iov_base = sldns_buffer_begin(data->pkt_buf);
1672 		iov[0].iov_len = sldns_buffer_remaining(data->pkt_buf);
1673 		msg.msg_iov = iov;
1674 		msg.msg_iovlen = 1;
1675 		msg.msg_control = ancil.buf;
1676 #ifndef S_SPLINT_S
1677 		msg.msg_controllen = sizeof(ancil.buf);
1678 #endif /* S_SPLINT_S */
1679 		msg.msg_flags = 0;
1680 
1681 		rcv = recvmsg(data->fd, &msg, MSG_DONTWAIT);
1682 		if(rcv == -1) {
1683 			if(errno == EINTR || errno == EAGAIN)
1684 				break;
1685 			log_err_addr("doq recvmsg", strerror(errno),
1686 				&data->dest_addr, sizeof(data->dest_addr_len));
1687 			break;
1688 		}
1689 
1690 		pi.ecn = msghdr_get_ecn(&msg, addr.ss_family);
1691 		verbose(1, "recvmsg %d ecn=0x%x", (int)rcv, (int)pi.ecn);
1692 
1693 		memset(&path, 0, sizeof(path));
1694 		path.local.addr = (void*)&data->local_addr;
1695 		path.local.addrlen = data->local_addr_len;
1696 		path.remote.addr = (void*)msg.msg_name;
1697 		path.remote.addrlen = msg.msg_namelen;
1698 		rv = ngtcp2_conn_read_pkt(data->conn, &path, &pi,
1699 			iov[0].iov_base, rcv, get_timestamp_nanosec());
1700 		if(rv != 0) {
1701 			log_err("ngtcp2_conn_read_pkt failed: %s",
1702 				ngtcp2_strerror(rv));
1703 			if(
1704 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1705 				data->ccerr.error_code == 0
1706 #else
1707 				data->last_error.error_code == 0
1708 #endif
1709 				) {
1710 				if(rv == NGTCP2_ERR_CRYPTO) {
1711 					/* in picotls the tls alert may need
1712 					 * to be copied, but this is with
1713 					 * openssl. And we have the value
1714 					 * data.tls_alert. */
1715 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1716 					ngtcp2_ccerr_set_tls_alert(
1717 						&data->ccerr, data->tls_alert,
1718 						NULL, 0);
1719 #else
1720 					ngtcp2_connection_close_error_set_transport_error_tls_alert(
1721 						&data->last_error,
1722 						data->tls_alert, NULL, 0);
1723 #endif
1724 				} else {
1725 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1726 					ngtcp2_ccerr_set_liberr(&data->ccerr,
1727 						rv, NULL, 0);
1728 #else
1729 					ngtcp2_connection_close_error_set_transport_error_liberr(
1730 						&data->last_error, rv, NULL,
1731 						0);
1732 #endif
1733 				}
1734 			}
1735 			disconnect(data);
1736 			return;
1737 		}
1738 	}
1739 
1740 	update_timer(data);
1741 }
1742 
1743 /** the write of this query has completed, it has spooled to packets,
1744  * set it to have the write done and move it to the list of receive streams. */
1745 static void
1746 query_write_is_done(struct doq_client_data* data,
1747 	struct doq_client_stream* str)
1748 {
1749 	if(verbosity > 0) {
1750 		char* logs = client_stream_string(str);
1751 		verbose(1, "query %s write is done", logs);
1752 		free(logs);
1753 	}
1754 	str->write_is_done = 1;
1755 	stream_list_move(str, data->query_list_send, data->query_list_receive);
1756 }
1757 
1758 /** write the data streams, if possible */
1759 static int
1760 write_streams(struct doq_client_data* data)
1761 {
1762 	ngtcp2_path_storage ps;
1763 	ngtcp2_tstamp ts = get_timestamp_nanosec();
1764 	struct doq_client_stream* str, *next;
1765 	uint32_t flags;
1766 	/* number of bytes that can be sent without packet pacing */
1767 	size_t send_quantum = ngtcp2_conn_get_send_quantum(data->conn);
1768 	/* Overhead is the stream overhead of adding a header onto the data,
1769 	 * this make sure the number of bytes to send in data bytes plus
1770 	 * the overhead overshoots the target quantum by a smaller margin,
1771 	 * and then it stops sending more bytes. With zero it would overshoot
1772 	 * more, an accurate number would not overshoot. It is based on the
1773 	 * stream frame header size. */
1774 	size_t accumulated_send = 0, overhead_stream = 24, overhead_pkt = 60,
1775 		max_packet_size = 1200;
1776 	size_t num_packets = 0, max_packets = 65535;
1777 	ngtcp2_path_storage_zero(&ps);
1778 	str = data->query_list_send->first;
1779 
1780 	if(data->cc_algo != NGTCP2_CC_ALGO_BBR
1781 #ifdef NGTCP2_CC_ALGO_BBR_V2
1782 		&& data->cc_algo != NGTCP2_CC_ALGO_BBR_V2
1783 #endif
1784 #ifdef NGTCP2_CC_ALGO_BBR2
1785 		&& data->cc_algo != NGTCP2_CC_ALGO_BBR2
1786 #endif
1787 		) {
1788 		/* If we do not have a packet pacing congestion control
1789 		 * algorithm, limit the number of packets. */
1790 		max_packets = 10;
1791 	}
1792 
1793 	/* loop like this, because at the start, the send list is empty,
1794 	 * and we want to send handshake packets. But when there is a
1795 	 * send_list, loop through that. */
1796 	for(;;) {
1797 		int64_t stream_id;
1798 		ngtcp2_pkt_info pi;
1799 		ngtcp2_vec datav[2];
1800 		size_t datav_count = 0;
1801 		int fin;
1802 		ngtcp2_ssize ret;
1803 		ngtcp2_ssize ndatalen = 0;
1804 		int send_is_blocked = 0;
1805 
1806 		if(str) {
1807 			/* pick up next in case this one is deleted */
1808 			next = str->next;
1809 			if(verbosity > 0) {
1810 				char* logs = client_stream_string(str);
1811 				verbose(1, "query %s write stream", logs);
1812 				free(logs);
1813 			}
1814 			stream_id = str->stream_id;
1815 			fin = 1;
1816 			if(str->nwrite < 2) {
1817 				str->data_tcplen = htons(str->data_len);
1818 				datav[0].base = ((uint8_t*)&str->data_tcplen)+str->nwrite;
1819 				datav[0].len = 2-str->nwrite;
1820 				datav[1].base = str->data;
1821 				datav[1].len = str->data_len;
1822 				datav_count = 2;
1823 			} else {
1824 				datav[0].base = str->data + (str->nwrite-2);
1825 				datav[0].len = str->data_len - (str->nwrite-2);
1826 				datav_count = 1;
1827 			}
1828 		} else {
1829 			next = NULL;
1830 			verbose(1, "write stream -1.");
1831 			stream_id = -1;
1832 			fin = 0;
1833 			datav[0].base = NULL;
1834 			datav[0].len = 0;
1835 			datav_count = 1;
1836 		}
1837 
1838 		/* Does the first data entry fit into the send quantum? */
1839 		/* Check if the data size sent, with a max of one full packet,
1840 		 * with added stream header and packet header is allowed
1841 		 * within the send quantum number of bytes. If not, it does
1842 		 * not fit, and wait. */
1843 		if(accumulated_send == 0 && ((datav_count == 1 &&
1844 			(datav[0].len>max_packet_size?max_packet_size:
1845 			datav[0].len)+overhead_stream+overhead_pkt >
1846 			send_quantum) ||
1847 			(datav_count == 2 &&
1848 			(datav[0].len+datav[1].len>max_packet_size?
1849 			max_packet_size:datav[0].len+datav[1].len)
1850 			+overhead_stream+overhead_pkt > send_quantum))) {
1851 			/* congestion limited */
1852 			ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
1853 			event_change_write(data, 0);
1854 			/* update the timer to wait until it is possible to
1855 			 * write again */
1856 			update_timer(data);
1857 			return 0;
1858 		}
1859 		flags = 0;
1860 		if(str && str->next != NULL) {
1861 			/* Coalesce more data from more streams into this
1862 			 * packet, if possible */
1863 			/* There is more than one data entry in this send
1864 			 * quantum, does the next one fit in the quantum? */
1865 			size_t this_send, possible_next_send;
1866 			if(datav_count == 1)
1867 				this_send = datav[0].len;
1868 			else	this_send = datav[0].len + datav[1].len;
1869 			if(this_send > max_packet_size)
1870 				this_send = max_packet_size;
1871 			if(str->next->nwrite < 2)
1872 				possible_next_send = (2-str->next->nwrite) +
1873 					str->next->data_len;
1874 			else	possible_next_send = str->next->data_len -
1875 					(str->next->nwrite - 2);
1876 			if(possible_next_send > max_packet_size)
1877 				possible_next_send = max_packet_size;
1878 			/* Check if the data lengths that writev returned
1879 			 * with stream headers added up so far, in
1880 			 * accumulated_send, with added the data length
1881 			 * of this send, with a max of one full packet, and
1882 			 * the data length of the next possible send, with
1883 			 * a max of one full packet, with a stream header for
1884 			 * this_send and a stream header for the next possible
1885 			 * send and a packet header, fit in the send quantum
1886 			 * number of bytes. If so, ask to add more content
1887 			 * to the packet with the more flag. */
1888 			if(accumulated_send + this_send + possible_next_send
1889 				+2*overhead_stream+ overhead_pkt < send_quantum)
1890 				flags |= NGTCP2_WRITE_STREAM_FLAG_MORE;
1891 		}
1892 		if(fin) {
1893 			/* This is the final part of data for this stream */
1894 			flags |= NGTCP2_WRITE_STREAM_FLAG_FIN;
1895 		}
1896 		sldns_buffer_clear(data->pkt_buf);
1897 		ret = ngtcp2_conn_writev_stream(data->conn, &ps.path, &pi,
1898 			sldns_buffer_begin(data->pkt_buf),
1899 			sldns_buffer_remaining(data->pkt_buf), &ndatalen,
1900 			flags, stream_id, datav, datav_count, ts);
1901 		if(ret < 0) {
1902 			if(ret == NGTCP2_ERR_WRITE_MORE) {
1903 				if(str) {
1904 					str->nwrite += ndatalen;
1905 					if(str->nwrite >= str->data_len+2)
1906 						query_write_is_done(data, str);
1907 					str = next;
1908 					accumulated_send += ndatalen + overhead_stream;
1909 					continue;
1910 				}
1911 			}
1912 			log_err("ngtcp2_conn_writev_stream failed: %s",
1913 				ngtcp2_strerror(ret));
1914 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
1915 			ngtcp2_ccerr_set_liberr(&data->ccerr, ret, NULL, 0);
1916 #else
1917 			ngtcp2_connection_close_error_set_transport_error_liberr(
1918 				&data->last_error, ret, NULL, 0);
1919 #endif
1920 			disconnect(data);
1921 			return 0;
1922 		}
1923 		verbose(1, "writev_stream pkt size %d ndatawritten %d",
1924 			(int)ret, (int)ndatalen);
1925 		if(ndatalen >= 0 && str) {
1926 			/* add the new write offset */
1927 			str->nwrite += ndatalen;
1928 			if(str->nwrite >= str->data_len+2)
1929 				query_write_is_done(data, str);
1930 		}
1931 		if(ret == 0) {
1932 			/* congestion limited */
1933 			ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
1934 			event_change_write(data, 0);
1935 			/* update the timer to wait until it is possible to
1936 			 * write again */
1937 			update_timer(data);
1938 			return 0;
1939 		}
1940 		if(!doq_client_send_pkt(data, pi.ecn,
1941 			sldns_buffer_begin(data->pkt_buf), ret, 0,
1942 			&send_is_blocked)) {
1943 			if(send_is_blocked) {
1944 				/* Blocked packet, wait until it is possible
1945 				 * to write again and also set a timer. */
1946 				event_change_write(data, 1);
1947 				update_timer(data);
1948 				return 0;
1949 			}
1950 			/* Packet could not be sent. Like lost and timeout. */
1951 			ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
1952 			event_change_write(data, 0);
1953 			update_timer(data);
1954 			return 0;
1955 		}
1956 		/* continue */
1957 		if((size_t)ret >= send_quantum)
1958 			break;
1959 		send_quantum -= ret;
1960 		accumulated_send = 0;
1961 		str = next;
1962 		if(str == NULL)
1963 			break;
1964 		if(++num_packets == max_packets)
1965 			break;
1966 	}
1967 	ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
1968 	event_change_write(data, 1);
1969 	return 1;
1970 }
1971 
1972 /** send the blocked packet now that the stream is writable again. */
1973 static int
1974 send_blocked_pkt(struct doq_client_data* data)
1975 {
1976 	ngtcp2_tstamp ts = get_timestamp_nanosec();
1977 	int send_is_blocked = 0;
1978 	if(!doq_client_send_pkt(data, data->blocked_pkt_pi.ecn,
1979 		sldns_buffer_begin(data->pkt_buf),
1980 		sldns_buffer_limit(data->pkt_buf), 1, &send_is_blocked)) {
1981 		if(send_is_blocked) {
1982 			/* Send was blocked, again. Wait, again to retry. */
1983 			event_change_write(data, 1);
1984 			/* make sure the timer is set while waiting */
1985 			update_timer(data);
1986 			return 0;
1987 		}
1988 		/* The packed could not be sent. Like it was lost, timeout. */
1989 		data->have_blocked_pkt = 0;
1990 		ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
1991 		event_change_write(data, 0);
1992 		update_timer(data);
1993 		return 0;
1994 	}
1995 	/* The blocked packet has been sent, the holding buffer can be
1996 	 * cleared. */
1997 	data->have_blocked_pkt = 0;
1998 	ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
1999 	return 1;
2000 }
2001 
2002 /** perform write operations, if any, on fd */
2003 static void
2004 on_write(struct doq_client_data* data)
2005 {
2006 	if(data->have_blocked_pkt) {
2007 		if(!send_blocked_pkt(data))
2008 			return;
2009 	}
2010 	if(
2011 #ifdef HAVE_NGTCP2_CONN_IN_CLOSING_PERIOD
2012 		ngtcp2_conn_in_closing_period(data->conn)
2013 #else
2014 		ngtcp2_conn_is_in_closing_period(data->conn)
2015 #endif
2016 		)
2017 		return;
2018 	if(!write_streams(data))
2019 		return;
2020 	update_timer(data);
2021 }
2022 
2023 /** callback for main listening file descriptor */
2024 void
2025 doq_client_event_cb(int ATTR_UNUSED(fd), short bits, void* arg)
2026 {
2027 	struct doq_client_data* data = (struct doq_client_data*)arg;
2028 	verbose(1, "doq_client_event_cb %s%s%s",
2029 		((bits&UB_EV_READ)!=0?"EV_READ":""),
2030 		((bits&(UB_EV_READ|UB_EV_WRITE))==(UB_EV_READ|UB_EV_WRITE)?
2031 		" ":""),
2032 		((bits&UB_EV_WRITE)!=0?"EV_WRITE":""));
2033 	if((bits&UB_EV_READ)) {
2034 		on_read(data);
2035 	}
2036 	/* Perform the write operation anyway. The read operation may
2037 	 * have produced data, or there is content waiting and it is possible
2038 	 * to write that. */
2039 	on_write(data);
2040 }
2041 
2042 /** read the TLS session from file */
2043 static int
2044 early_data_setup_session(struct doq_client_data* data)
2045 {
2046 	SSL_SESSION* session;
2047 	BIO* f = BIO_new_file(data->session_file, "r");
2048 	if(f == NULL) {
2049 		if(errno == ENOENT) {
2050 			verbose(1, "session file %s does not exist",
2051 				data->session_file);
2052 			return 0;
2053 		}
2054 		log_err("Could not read %s: %s", data->session_file,
2055 			strerror(errno));
2056 		return 0;
2057 	}
2058 	session = PEM_read_bio_SSL_SESSION(f, NULL, 0, NULL);
2059 	if(session == NULL) {
2060 		log_crypto_err("Could not read session file with PEM_read_bio_SSL_SESSION");
2061 		BIO_free(f);
2062 		return 0;
2063 	}
2064 	BIO_free(f);
2065 	if(!SSL_set_session(data->ssl, session)) {
2066 		log_crypto_err("Could not SSL_set_session");
2067 		SSL_SESSION_free(session);
2068 		return 0;
2069 	}
2070 	if(SSL_SESSION_get_max_early_data(session) == 0) {
2071 		log_err("TLS session early data is 0");
2072 		SSL_SESSION_free(session);
2073 		return 0;
2074 	}
2075 	SSL_set_quic_early_data_enabled(data->ssl, 1);
2076 	SSL_SESSION_free(session);
2077 	return 1;
2078 }
2079 
2080 #ifndef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
2081 /** parse one line from the transport file */
2082 static int
2083 transport_parse_line(struct ngtcp2_transport_params* params, char* line)
2084 {
2085 	if(strncmp(line, "initial_max_streams_bidi=", 25) == 0) {
2086 		params->initial_max_streams_bidi = atoi(line+25);
2087 		return 1;
2088 	}
2089 	if(strncmp(line, "initial_max_streams_uni=", 24) == 0) {
2090 		params->initial_max_streams_uni = atoi(line+24);
2091 		return 1;
2092 	}
2093 	if(strncmp(line, "initial_max_stream_data_bidi_local=", 35) == 0) {
2094 		params->initial_max_stream_data_bidi_local = atoi(line+35);
2095 		return 1;
2096 	}
2097 	if(strncmp(line, "initial_max_stream_data_bidi_remote=", 36) == 0) {
2098 		params->initial_max_stream_data_bidi_remote = atoi(line+36);
2099 		return 1;
2100 	}
2101 	if(strncmp(line, "initial_max_stream_data_uni=", 28) == 0) {
2102 		params->initial_max_stream_data_uni = atoi(line+28);
2103 		return 1;
2104 	}
2105 	if(strncmp(line, "initial_max_data=", 17) == 0) {
2106 		params->initial_max_data = atoi(line+17);
2107 		return 1;
2108 	}
2109 	if(strncmp(line, "active_connection_id_limit=", 27) == 0) {
2110 		params->active_connection_id_limit = atoi(line+27);
2111 		return 1;
2112 	}
2113 	if(strncmp(line, "max_datagram_frame_size=", 24) == 0) {
2114 		params->max_datagram_frame_size = atoi(line+24);
2115 		return 1;
2116 	}
2117 	return 0;
2118 }
2119 #endif /* HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS */
2120 
2121 /** setup the early data transport file and read it */
2122 static int
2123 early_data_setup_transport(struct doq_client_data* data)
2124 {
2125 #ifdef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
2126 	FILE* in;
2127 	uint8_t buf[1024];
2128 	size_t len;
2129 	int rv;
2130 	in = fopen(data->transport_file, "r");
2131 	if(!in) {
2132 		if(errno == ENOENT) {
2133 			verbose(1, "transport file %s does not exist",
2134 				data->transport_file);
2135 			return 0;
2136 		}
2137 		perror(data->transport_file);
2138 		return 0;
2139 	}
2140 	len = fread(buf, 1, sizeof(buf), in);
2141 	if(ferror(in)) {
2142 		log_err("%s: read failed: %s", data->transport_file,
2143 			strerror(errno));
2144 		fclose(in);
2145 		return 0;
2146 	}
2147 	fclose(in);
2148 	rv = ngtcp2_conn_decode_and_set_0rtt_transport_params(data->conn,
2149 		buf, len);
2150 	if(rv != 0) {
2151 		log_err("ngtcp2_conn_decode_and_set_0rtt_transport_params failed: %s",
2152 			ngtcp2_strerror(rv));
2153 		return 0;
2154 	}
2155 	return 1;
2156 #else
2157 	FILE* in;
2158 	char buf[1024];
2159 	struct ngtcp2_transport_params params;
2160 	memset(&params, 0, sizeof(params));
2161 	in = fopen(data->transport_file, "r");
2162 	if(!in) {
2163 		if(errno == ENOENT) {
2164 			verbose(1, "transport file %s does not exist",
2165 				data->transport_file);
2166 			return 0;
2167 		}
2168 		perror(data->transport_file);
2169 		return 0;
2170 	}
2171 	while(!feof(in)) {
2172 		if(!fgets(buf, sizeof(buf), in)) {
2173 			log_err("%s: read failed: %s", data->transport_file,
2174 				strerror(errno));
2175 			fclose(in);
2176 			return 0;
2177 		}
2178 		if(!transport_parse_line(&params, buf)) {
2179 			log_err("%s: could not parse line '%s'",
2180 				data->transport_file, buf);
2181 			fclose(in);
2182 			return 0;
2183 		}
2184 	}
2185 	fclose(in);
2186 	ngtcp2_conn_set_early_remote_transport_params(data->conn, &params);
2187 #endif
2188 	return 1;
2189 }
2190 
2191 /** setup for early data, read the transport file and session file */
2192 static void
2193 early_data_setup(struct doq_client_data* data)
2194 {
2195 	if(!early_data_setup_session(data)) {
2196 		verbose(1, "TLS session resumption failed, early data is disabled");
2197 		data->early_data_enabled = 0;
2198 		return;
2199 	}
2200 	if(!early_data_setup_transport(data)) {
2201 		verbose(1, "Transport parameters set failed, early data is disabled");
2202 		data->early_data_enabled = 0;
2203 		return;
2204 	}
2205 }
2206 
2207 /** start the early data transmission */
2208 static void
2209 early_data_start(struct doq_client_data* data)
2210 {
2211 	query_streams_start(data);
2212 	on_write(data);
2213 }
2214 
2215 /** create doq_client_data */
2216 static struct doq_client_data*
2217 create_doq_client_data(const char* svr, int port, struct ub_event_base* base,
2218 	const char* transport_file, const char* session_file, int quiet)
2219 {
2220 	struct doq_client_data* data;
2221 	data = calloc(1, sizeof(*data));
2222 	if(!data) fatal_exit("calloc failed: out of memory");
2223 	data->base = base;
2224 	data->rnd = ub_initstate(NULL);
2225 	if(!data->rnd) fatal_exit("ub_initstate failed: out of memory");
2226 	data->svr = svr;
2227 	get_dest_addr(data, svr, port);
2228 	data->port = port;
2229 	data->quiet = quiet;
2230 	data->pkt_buf = sldns_buffer_new(65552);
2231 	if(!data->pkt_buf)
2232 		fatal_exit("sldns_buffer_new failed: out of memory");
2233 	data->blocked_pkt = sldns_buffer_new(65552);
2234 	if(!data->blocked_pkt)
2235 		fatal_exit("sldns_buffer_new failed: out of memory");
2236 	data->fd = open_svr_udp(data);
2237 	get_local_addr(data);
2238 	data->conn = conn_client_setup(data);
2239 #ifdef HAVE_NGTCP2_CCERR_DEFAULT
2240 	ngtcp2_ccerr_default(&data->ccerr);
2241 #else
2242 	ngtcp2_connection_close_error_default(&data->last_error);
2243 #endif
2244 	data->transport_file = transport_file;
2245 	data->session_file = session_file;
2246 	if(data->transport_file && data->session_file)
2247 		data->early_data_enabled = 1;
2248 
2249 	generate_static_secret(data, 32);
2250 	data->ctx = ctx_client_setup();
2251 	if(data->session_file) {
2252 		SSL_CTX_set_session_cache_mode(data->ctx,
2253 			SSL_SESS_CACHE_CLIENT |
2254 			SSL_SESS_CACHE_NO_INTERNAL_STORE);
2255 		SSL_CTX_sess_set_new_cb(data->ctx, new_session_cb);
2256 	}
2257 	data->ssl = ssl_client_setup(data);
2258 	ngtcp2_conn_set_tls_native_handle(data->conn, data->ssl);
2259 	if(data->early_data_enabled)
2260 		early_data_setup(data);
2261 
2262 	data->ev = ub_event_new(base, data->fd, UB_EV_READ | UB_EV_WRITE |
2263 		UB_EV_PERSIST, doq_client_event_cb, data);
2264 	if(!data->ev) {
2265 		fatal_exit("could not ub_event_new");
2266 	}
2267 	if(ub_event_add(data->ev, NULL) != 0) {
2268 		fatal_exit("could not ub_event_add");
2269 	}
2270 	data->expire_timer = ub_event_new(data->base, -1,
2271 		UB_EV_TIMEOUT, &doq_client_timer_cb, data);
2272 	if(!data->expire_timer)
2273 		fatal_exit("could not ub_event_new");
2274 	data->query_list_start = stream_list_create();
2275 	data->query_list_send = stream_list_create();
2276 	data->query_list_receive = stream_list_create();
2277 	data->query_list_stop = stream_list_create();
2278 	return data;
2279 }
2280 
2281 /** delete doq_client_data */
2282 static void
2283 delete_doq_client_data(struct doq_client_data* data)
2284 {
2285 	if(!data)
2286 		return;
2287 #if defined(NGTCP2_USE_GENERIC_SOCKADDR) || defined(NGTCP2_USE_GENERIC_IPV6_SOCKADDR)
2288 	if(data->conn && data->dest_addr_len != 0) {
2289 		if(addr_is_ip6(&data->dest_addr, data->dest_addr_len)) {
2290 #  if defined(NGTCP2_USE_GENERIC_SOCKADDR) || defined(NGTCP2_USE_GENERIC_IPV6_SOCKADDR)
2291 			const struct ngtcp2_path* path6 = ngtcp2_conn_get_path(data->conn);
2292 			free(path6->local.addr);
2293 			free(path6->remote.addr);
2294 #  endif
2295 		} else {
2296 #  if defined(NGTCP2_USE_GENERIC_SOCKADDR)
2297 			const struct ngtcp2_path* path = ngtcp2_conn_get_path(data->conn);
2298 			free(path->local.addr);
2299 			free(path->remote.addr);
2300 #  endif
2301 		}
2302 	}
2303 #endif
2304 	ngtcp2_conn_del(data->conn);
2305 	SSL_free(data->ssl);
2306 	sldns_buffer_free(data->pkt_buf);
2307 	sldns_buffer_free(data->blocked_pkt);
2308 	if(data->fd != -1)
2309 		sock_close(data->fd);
2310 	SSL_CTX_free(data->ctx);
2311 	stream_list_free(data->query_list_start);
2312 	stream_list_free(data->query_list_send);
2313 	stream_list_free(data->query_list_receive);
2314 	stream_list_free(data->query_list_stop);
2315 	ub_randfree(data->rnd);
2316 	if(data->ev) {
2317 		ub_event_del(data->ev);
2318 		ub_event_free(data->ev);
2319 	}
2320 	if(data->expire_timer_added)
2321 		ub_timer_del(data->expire_timer);
2322 	ub_event_free(data->expire_timer);
2323 	free(data->static_secret_data);
2324 	free(data);
2325 }
2326 
2327 /** create the event base that registers events and timers */
2328 static struct ub_event_base*
2329 create_event_base(time_t* secs, struct timeval* now)
2330 {
2331 	struct ub_event_base* base;
2332 	const char *evnm="event", *evsys="", *evmethod="";
2333 
2334 	memset(now, 0, sizeof(*now));
2335 	base = ub_default_event_base(1, secs, now);
2336 	if(!base) fatal_exit("could not create ub_event base");
2337 
2338 	ub_get_event_sys(base, &evnm, &evsys, &evmethod);
2339 	if(verbosity) log_info("%s %s uses %s method", evnm, evsys, evmethod);
2340 
2341 	return base;
2342 }
2343 
2344 /** enter a query into the query list */
2345 static void
2346 client_enter_query_buf(struct doq_client_data* data, struct sldns_buffer* buf)
2347 {
2348 	struct doq_client_stream* str;
2349 	str = client_stream_create(buf);
2350 	if(!str)
2351 		fatal_exit("client_stream_create failed: out of memory");
2352 	stream_list_append(data->query_list_start, str);
2353 }
2354 
2355 /** enter the queries into the query list */
2356 static void
2357 client_enter_queries(struct doq_client_data* data, char** qs, int count)
2358 {
2359 	int i;
2360 	for(i=0; i<count; i+=3) {
2361 		struct sldns_buffer* buf = NULL;
2362 		buf = make_query(qs[i], qs[i+1], qs[i+2]);
2363 		if(verbosity > 0) {
2364 			char* str;
2365 			log_buf(1, "send query", buf);
2366 			str = sldns_wire2str_pkt(sldns_buffer_begin(buf),
2367 				sldns_buffer_limit(buf));
2368 			if(!str) verbose(1, "could not sldns_wire2str_pkt");
2369 			else verbose(1, "send query:\n%s", str);
2370 			free(str);
2371 		}
2372 		client_enter_query_buf(data, buf);
2373 		sldns_buffer_free(buf);
2374 	}
2375 }
2376 
2377 /** run the dohclient queries */
2378 static void run(const char* svr, int port, char** qs, int count,
2379 	const char* transport_file, const char* session_file, int quiet)
2380 {
2381 	time_t secs = 0;
2382 	struct timeval now;
2383 	struct ub_event_base* base;
2384 	struct doq_client_data* data;
2385 
2386 	/* setup */
2387 	base = create_event_base(&secs, &now);
2388 	data = create_doq_client_data(svr, port, base, transport_file,
2389 		session_file, quiet);
2390 	client_enter_queries(data, qs, count);
2391 	if(data->early_data_enabled)
2392 		early_data_start(data);
2393 
2394 	/* run the queries */
2395 	ub_event_base_dispatch(base);
2396 
2397 	/* cleanup */
2398 	delete_doq_client_data(data);
2399 	ub_event_base_free(base);
2400 }
2401 #endif /* HAVE_NGTCP2 */
2402 
2403 #ifdef HAVE_NGTCP2
2404 /** getopt global, in case header files fail to declare it. */
2405 extern int optind;
2406 /** getopt global, in case header files fail to declare it. */
2407 extern char* optarg;
2408 int main(int ATTR_UNUSED(argc), char** ATTR_UNUSED(argv))
2409 {
2410 	int c;
2411 	int port = UNBOUND_DNS_OVER_QUIC_PORT, quiet = 0;
2412 	const char* svr = "127.0.0.1", *transport_file = NULL,
2413 		*session_file = NULL;
2414 #ifdef USE_WINSOCK
2415 	WSADATA wsa_data;
2416 	if(WSAStartup(MAKEWORD(2,2), &wsa_data) != 0) {
2417 		printf("WSAStartup failed\n");
2418 		return 1;
2419 	}
2420 #endif
2421 	checklock_set_output_name("ublocktrace-doqclient");
2422 	checklock_start();
2423 	log_init(0, 0, 0);
2424 	log_ident_set("doqclient");
2425 
2426 	while((c=getopt(argc, argv, "hp:qs:vx:y:")) != -1) {
2427 		switch(c) {
2428 			case 'p':
2429 				if(atoi(optarg)==0 && strcmp(optarg,"0")!=0) {
2430 					printf("error parsing port, "
2431 					    "number expected: %s\n", optarg);
2432 					return 1;
2433 				}
2434 				port = atoi(optarg);
2435 				break;
2436 			case 'q':
2437 				quiet++;
2438 				break;
2439 			case 's':
2440 				svr = optarg;
2441 				break;
2442 			case 'v':
2443 				verbosity++;
2444 				break;
2445 			case 'x':
2446 				transport_file = optarg;
2447 				break;
2448 			case 'y':
2449 				session_file = optarg;
2450 				break;
2451 			case 'h':
2452 			case '?':
2453 			default:
2454 				usage(argv);
2455 		}
2456 	}
2457 
2458 	argc -= optind;
2459 	argv += optind;
2460 
2461 	if(argc%3!=0) {
2462 		printf("Invalid input. Specify qname, qtype, and qclass.\n");
2463 		return 1;
2464 	}
2465 	if(port == 53) {
2466 		printf("Error: port number 53 not for DNS over QUIC. Port number 53 is not allowed to be used with DNS over QUIC. It is used for DNS datagrams.\n");
2467 		return 1;
2468 	}
2469 
2470 	run(svr, port, argv, argc, transport_file, session_file, quiet);
2471 
2472 	checklock_stop();
2473 #ifdef USE_WINSOCK
2474 	WSACleanup();
2475 #endif
2476 	return 0;
2477 }
2478 #else /* HAVE_NGTCP2 */
2479 int main(int ATTR_UNUSED(argc), char** ATTR_UNUSED(argv))
2480 {
2481 	printf("Compiled without ngtcp2 for QUIC, cannot run doqclient.\n");
2482 	return 1;
2483 }
2484 #endif /* HAVE_NGTCP2 */
2485 
2486 /***--- definitions to make fptr_wlist work. ---***/
2487 /* These are callbacks, similar to smallapp callbacks, except the debug
2488  * tool callbacks are not in it */
2489 struct tube;
2490 struct query_info;
2491 #include "util/data/packed_rrset.h"
2492 #include "daemon/worker.h"
2493 #include "daemon/remote.h"
2494 #include "util/fptr_wlist.h"
2495 #include "libunbound/context.h"
2496 
2497 void worker_handle_control_cmd(struct tube* ATTR_UNUSED(tube),
2498 	uint8_t* ATTR_UNUSED(buffer), size_t ATTR_UNUSED(len),
2499 	int ATTR_UNUSED(error), void* ATTR_UNUSED(arg))
2500 {
2501 	log_assert(0);
2502 }
2503 
2504 int worker_handle_request(struct comm_point* ATTR_UNUSED(c),
2505 	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
2506         struct comm_reply* ATTR_UNUSED(repinfo))
2507 {
2508 	log_assert(0);
2509 	return 0;
2510 }
2511 
2512 int worker_handle_service_reply(struct comm_point* ATTR_UNUSED(c),
2513 	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
2514         struct comm_reply* ATTR_UNUSED(reply_info))
2515 {
2516 	log_assert(0);
2517 	return 0;
2518 }
2519 
2520 int remote_accept_callback(struct comm_point* ATTR_UNUSED(c),
2521 	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
2522         struct comm_reply* ATTR_UNUSED(repinfo))
2523 {
2524 	log_assert(0);
2525 	return 0;
2526 }
2527 
2528 int remote_control_callback(struct comm_point* ATTR_UNUSED(c),
2529 	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
2530         struct comm_reply* ATTR_UNUSED(repinfo))
2531 {
2532 	log_assert(0);
2533 	return 0;
2534 }
2535 
2536 void worker_sighandler(int ATTR_UNUSED(sig), void* ATTR_UNUSED(arg))
2537 {
2538 	log_assert(0);
2539 }
2540 
2541 struct outbound_entry* worker_send_query(
2542 	struct query_info* ATTR_UNUSED(qinfo), uint16_t ATTR_UNUSED(flags),
2543 	int ATTR_UNUSED(dnssec), int ATTR_UNUSED(want_dnssec),
2544 	int ATTR_UNUSED(nocaps), int ATTR_UNUSED(check_ratelimit),
2545 	struct sockaddr_storage* ATTR_UNUSED(addr),
2546 	socklen_t ATTR_UNUSED(addrlen), uint8_t* ATTR_UNUSED(zone),
2547 	size_t ATTR_UNUSED(zonelen), int ATTR_UNUSED(tcp_upstream),
2548 	int ATTR_UNUSED(ssl_upstream), char* ATTR_UNUSED(tls_auth_name),
2549 	struct module_qstate* ATTR_UNUSED(q), int* ATTR_UNUSED(was_ratelimited))
2550 {
2551 	log_assert(0);
2552 	return 0;
2553 }
2554 
2555 #ifdef UB_ON_WINDOWS
2556 void
2557 worker_win_stop_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev), void*
2558 	ATTR_UNUSED(arg)) {
2559 	log_assert(0);
2560 }
2561 
2562 void
2563 wsvc_cron_cb(void* ATTR_UNUSED(arg))
2564 {
2565 	log_assert(0);
2566 }
2567 #endif /* UB_ON_WINDOWS */
2568 
2569 void
2570 worker_alloc_cleanup(void* ATTR_UNUSED(arg))
2571 {
2572 	log_assert(0);
2573 }
2574 
2575 struct outbound_entry* libworker_send_query(
2576 	struct query_info* ATTR_UNUSED(qinfo), uint16_t ATTR_UNUSED(flags),
2577 	int ATTR_UNUSED(dnssec), int ATTR_UNUSED(want_dnssec),
2578 	int ATTR_UNUSED(nocaps), int ATTR_UNUSED(check_ratelimit),
2579 	struct sockaddr_storage* ATTR_UNUSED(addr),
2580 	socklen_t ATTR_UNUSED(addrlen), uint8_t* ATTR_UNUSED(zone),
2581 	size_t ATTR_UNUSED(zonelen), int ATTR_UNUSED(tcp_upstream),
2582 	int ATTR_UNUSED(ssl_upstream), char* ATTR_UNUSED(tls_auth_name),
2583 	struct module_qstate* ATTR_UNUSED(q), int* ATTR_UNUSED(was_ratelimited))
2584 {
2585 	log_assert(0);
2586 	return 0;
2587 }
2588 
2589 int libworker_handle_service_reply(struct comm_point* ATTR_UNUSED(c),
2590 	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
2591         struct comm_reply* ATTR_UNUSED(reply_info))
2592 {
2593 	log_assert(0);
2594 	return 0;
2595 }
2596 
2597 void libworker_handle_control_cmd(struct tube* ATTR_UNUSED(tube),
2598         uint8_t* ATTR_UNUSED(buffer), size_t ATTR_UNUSED(len),
2599         int ATTR_UNUSED(error), void* ATTR_UNUSED(arg))
2600 {
2601         log_assert(0);
2602 }
2603 
2604 void libworker_fg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode),
2605 	struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s),
2606 	char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited))
2607 {
2608 	log_assert(0);
2609 }
2610 
2611 void libworker_bg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode),
2612 	struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s),
2613 	char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited))
2614 {
2615 	log_assert(0);
2616 }
2617 
2618 void libworker_event_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode),
2619 	struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s),
2620 	char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited))
2621 {
2622 	log_assert(0);
2623 }
2624 
2625 int context_query_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
2626 {
2627 	log_assert(0);
2628 	return 0;
2629 }
2630 
2631 void worker_stat_timer_cb(void* ATTR_UNUSED(arg))
2632 {
2633 	log_assert(0);
2634 }
2635 
2636 void worker_probe_timer_cb(void* ATTR_UNUSED(arg))
2637 {
2638 	log_assert(0);
2639 }
2640 
2641 void worker_start_accept(void* ATTR_UNUSED(arg))
2642 {
2643 	log_assert(0);
2644 }
2645 
2646 void worker_stop_accept(void* ATTR_UNUSED(arg))
2647 {
2648 	log_assert(0);
2649 }
2650 
2651 /** keep track of lock id in lock-verify application */
2652 struct order_id {
2653         /** the thread id that created it */
2654         int thr;
2655         /** the instance number of creation */
2656         int instance;
2657 };
2658 
2659 int order_lock_cmp(const void* e1, const void* e2)
2660 {
2661         const struct order_id* o1 = e1;
2662         const struct order_id* o2 = e2;
2663         if(o1->thr < o2->thr) return -1;
2664         if(o1->thr > o2->thr) return 1;
2665         if(o1->instance < o2->instance) return -1;
2666         if(o1->instance > o2->instance) return 1;
2667         return 0;
2668 }
2669 
2670 int
2671 codeline_cmp(const void* a, const void* b)
2672 {
2673         return strcmp(a, b);
2674 }
2675 
2676 int replay_var_compare(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
2677 {
2678         log_assert(0);
2679         return 0;
2680 }
2681 
2682 void remote_get_opt_ssl(char* ATTR_UNUSED(str), void* ATTR_UNUSED(arg))
2683 {
2684         log_assert(0);
2685 }
2686 
2687 #ifdef USE_DNSTAP
2688 void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
2689 	void* ATTR_UNUSED(arg))
2690 {
2691 	log_assert(0);
2692 }
2693 #endif
2694 
2695 #ifdef USE_DNSTAP
2696 void dtio_mainfdcallback(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
2697 	void* ATTR_UNUSED(arg))
2698 {
2699 	log_assert(0);
2700 }
2701 #endif
2702 
2703 void fast_reload_service_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
2704 	void* ATTR_UNUSED(arg))
2705 {
2706 	log_assert(0);
2707 }
2708 
2709 int fast_reload_client_callback(struct comm_point* ATTR_UNUSED(c),
2710 	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
2711         struct comm_reply* ATTR_UNUSED(repinfo))
2712 {
2713 	log_assert(0);
2714 	return 0;
2715 }
2716