xref: /freebsd/crypto/openssl/demos/http3/ossl-nghttp3-demo-server.c (revision e7be843b4a162e68651d3911f0357ed464915629)
1 /*
2  * Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
3  *
4  * Licensed under the Apache License 2.0 (the "License").  You may not use
5  * this file except in compliance with the License.  You can obtain a copy
6  * in the file LICENSE in the source distribution or at
7  * https://www.openssl.org/source/license.html
8  */
9 #include <assert.h>
10 #include <netinet/in.h>
11 #include <nghttp3/nghttp3.h>
12 #include <openssl/err.h>
13 #include <openssl/quic.h>
14 #include <openssl/ssl.h>
15 #include <unistd.h>
16 #include <sys/stat.h>
17 #include <fcntl.h>
18 #include <sys/socket.h>
19 
20 #ifndef PATH_MAX
21 # define PATH_MAX 255
22 #endif
23 
24 #define nghttp3_arraylen(A) (sizeof(A) / sizeof(*(A)))
25 
26 /* The crappy test wants 20 bytes */
27 #define NULL_PAYLOAD "12345678901234567890"
28 static uint8_t *nulldata = (uint8_t *) NULL_PAYLOAD;
29 static size_t nulldata_sz = sizeof(NULL_PAYLOAD) - 1;
30 
31 /* The nghttp3 variable we need in the main part and read_from_ssl_ids */
32 static nghttp3_settings settings;
33 static const nghttp3_mem *mem;
34 static nghttp3_callbacks callbacks = {0};
35 
36 /* 3 streams created by the server and 4 by the client (one is bidi) */
37 struct ssl_id {
38     SSL *s;      /* the stream openssl uses in SSL_read(),  SSL_write etc */
39     uint64_t id; /* the stream identifier the nghttp3 uses */
40     int status;  /* 0 or one the below status and origin */
41 };
42 /* status and origin of the streams the possible values are: */
43 #define CLIENTUNIOPEN  0x01 /* unidirectional open by the client (2, 6 and 10) */
44 #define CLIENTCLOSED   0x02 /* closed by the client */
45 #define CLIENTBIDIOPEN 0x04 /* bidirectional open by the client (something like 0, 4, 8 ...) */
46 #define SERVERUNIOPEN  0x08 /* unidirectional open by the server (3, 7 and 11) */
47 #define SERVERCLOSED   0x10 /* closed by the server (us) */
48 #define TOBEREMOVED    0x20 /* marked for removing in read_from_ssl_ids, */
49                             /* it will be removed after processing all events */
50 #define ISLISTENER     0x40 /* the stream is a listener from SSL_new_listener() */
51 #define ISCONNECTION   0x80 /* the stream is a connection from SSL_accept_connection() */
52 
53 #define MAXSSL_IDS 20
54 #define MAXURL 255
55 
56 struct h3ssl {
57     struct ssl_id ssl_ids[MAXSSL_IDS];
58     int end_headers_received; /* h3 header received call back called */
59     int datadone;             /* h3 has given openssl all the data of the response */
60     int has_uni;              /* we have the 3 uni directional stream needed */
61     int close_done;           /* connection begins terminating EVENT_EC */
62     int close_wait;           /* we are waiting for a close or a new request */
63     int done;                 /* connection terminated EVENT_ECD, after EVENT_EC */
64     int new_conn;             /* a new connection has been received */
65     int received_from_two;    /* workaround for -607 on nghttp3_conn_read_stream on stream 2 */
66     int restart;              /* new request/response cycle started */
67     uint64_t id_bidi;         /* the id of the stream used to read request and send response */
68     char *fileprefix;         /* prefix of the directory to fetch files from */
69     char url[MAXURL];         /* url to serve the request */
70     uint8_t *ptr_data;        /* pointer to the data to send */
71     size_t ldata;             /* amount of bytes to send */
72     int offset_data;          /* offset to next data to send */
73 };
74 
make_nv(nghttp3_nv * nv,const char * name,const char * value)75 static void make_nv(nghttp3_nv *nv, const char *name, const char *value)
76 {
77     nv->name        = (uint8_t *)name;
78     nv->value       = (uint8_t *)value;
79     nv->namelen     = strlen(name);
80     nv->valuelen    = strlen(value);
81     nv->flags       = NGHTTP3_NV_FLAG_NONE;
82 }
83 
init_ids(struct h3ssl * h3ssl)84 static void init_ids(struct h3ssl *h3ssl)
85 {
86     struct ssl_id *ssl_ids;
87     int i;
88     char *prior_fileprefix = h3ssl->fileprefix;
89 
90     if (h3ssl->ptr_data != NULL && h3ssl->ptr_data != nulldata)
91         free(h3ssl->ptr_data);
92 
93     memset(h3ssl, 0, sizeof(struct h3ssl));
94 
95     ssl_ids = h3ssl->ssl_ids;
96     for (i = 0; i < MAXSSL_IDS; i++)
97         ssl_ids[i].id = UINT64_MAX;
98     h3ssl->id_bidi = UINT64_MAX;
99 
100     /* restore the fileprefix */
101     h3ssl->fileprefix = prior_fileprefix;
102 }
103 
reuse_h3ssl(struct h3ssl * h3ssl)104 static void reuse_h3ssl(struct h3ssl *h3ssl)
105 {
106     h3ssl->end_headers_received = 0;
107     h3ssl->datadone = 0;
108     h3ssl->close_done = 0;
109     h3ssl->close_wait = 0;
110     h3ssl->done = 0;
111     memset(h3ssl->url, '\0', sizeof(h3ssl->url));
112     if (h3ssl->ptr_data != NULL && h3ssl->ptr_data != nulldata)
113         free(h3ssl->ptr_data);
114     h3ssl->ptr_data = NULL;
115     h3ssl->offset_data = 0;
116     h3ssl->ldata = 0;
117 }
118 
add_id_status(uint64_t id,SSL * ssl,struct h3ssl * h3ssl,int status)119 static void add_id_status(uint64_t id, SSL *ssl, struct h3ssl *h3ssl, int status)
120 {
121     struct ssl_id *ssl_ids;
122     int i;
123 
124     ssl_ids = h3ssl->ssl_ids;
125     for (i = 0; i < MAXSSL_IDS; i++) {
126         if (ssl_ids[i].s == NULL) {
127             ssl_ids[i].s = ssl;
128             ssl_ids[i].id = id;
129             ssl_ids[i].status = status;
130             return;
131         }
132     }
133     printf("Oops too many streams to add!!!\n");
134     exit(1);
135 }
add_id(uint64_t id,SSL * ssl,struct h3ssl * h3ssl)136 static void add_id(uint64_t id, SSL *ssl, struct h3ssl *h3ssl)
137 {
138     add_id_status(id, ssl, h3ssl, 0);
139 }
140 
141 /* Add listener and connection */
add_ids_listener(SSL * ssl,struct h3ssl * h3ssl)142 static void add_ids_listener(SSL *ssl, struct h3ssl *h3ssl)
143 {
144     add_id_status(UINT64_MAX, ssl, h3ssl, ISLISTENER);
145 }
add_ids_connection(struct h3ssl * h3ssl,SSL * ssl)146 static void add_ids_connection(struct h3ssl *h3ssl, SSL *ssl)
147 {
148     add_id_status(UINT64_MAX, ssl, h3ssl, ISCONNECTION);
149 }
get_ids_connection(struct h3ssl * h3ssl)150 static SSL *get_ids_connection(struct h3ssl *h3ssl)
151 {
152     struct ssl_id *ssl_ids;
153     int i;
154 
155     ssl_ids = h3ssl->ssl_ids;
156     for (i = 0; i < MAXSSL_IDS; i++) {
157         if (ssl_ids[i].status & ISCONNECTION) {
158             printf("get_ids_connection\n");
159             return ssl_ids[i].s;
160         }
161     }
162     return NULL;
163 }
replace_ids_connection(struct h3ssl * h3ssl,SSL * oldstream,SSL * newstream)164 static void replace_ids_connection(struct h3ssl *h3ssl, SSL *oldstream, SSL *newstream)
165 {
166     struct ssl_id *ssl_ids;
167     int i;
168 
169     ssl_ids = h3ssl->ssl_ids;
170     for (i = 0; i < MAXSSL_IDS; i++) {
171         if (ssl_ids[i].status & ISCONNECTION && ssl_ids[i].s == oldstream) {
172             printf("replace_ids_connection\n");
173             ssl_ids[i].s = newstream;
174         }
175     }
176 }
177 
178 /* remove the ids marked for removal */
remove_marked_ids(struct h3ssl * h3ssl)179 static void remove_marked_ids(struct h3ssl *h3ssl)
180 {
181     struct ssl_id *ssl_ids;
182     int i;
183 
184     ssl_ids = h3ssl->ssl_ids;
185     for (i = 0; i < MAXSSL_IDS; i++) {
186         if (ssl_ids[i].status & TOBEREMOVED) {
187             printf("remove_id %llu\n", (unsigned long long) ssl_ids[i].id);
188             SSL_free(ssl_ids[i].s);
189             ssl_ids[i].s = NULL;
190             ssl_ids[i].id = UINT64_MAX;
191             ssl_ids[i].status = 0;
192             return;
193         }
194     }
195 }
196 
197 /* add the status bytes to the status */
set_id_status(uint64_t id,int status,struct h3ssl * h3ssl)198 static void set_id_status(uint64_t id, int status, struct h3ssl *h3ssl)
199 {
200     struct ssl_id *ssl_ids;
201     int i;
202 
203     ssl_ids = h3ssl->ssl_ids;
204     for (i = 0; i < MAXSSL_IDS; i++) {
205         if (ssl_ids[i].id == id) {
206             printf("set_id_status: %llu to %d\n", (unsigned long long) ssl_ids[i].id, status);
207             ssl_ids[i].status = ssl_ids[i].status | status;
208             return;
209         }
210     }
211     printf("Oops can't set status, can't find stream!!!\n");
212     assert(0);
213 }
get_id_status(uint64_t id,struct h3ssl * h3ssl)214 static int get_id_status(uint64_t id, struct h3ssl *h3ssl)
215 {
216     struct ssl_id *ssl_ids;
217     int i;
218 
219     ssl_ids = h3ssl->ssl_ids;
220     for (i = 0; i < MAXSSL_IDS; i++) {
221         if (ssl_ids[i].id == id) {
222             printf("get_id_status: %llu to %d\n",
223                    (unsigned long long) ssl_ids[i].id, ssl_ids[i].status);
224             return ssl_ids[i].status;
225         }
226     }
227     printf("Oops can't get status, can't find stream!!!\n");
228     assert(0);
229     return -1;
230 }
231 
232 /* check that all streams opened by the client are closed */
are_all_clientid_closed(struct h3ssl * h3ssl)233 static int are_all_clientid_closed(struct h3ssl *h3ssl)
234 {
235     struct ssl_id *ssl_ids;
236     int i;
237 
238     ssl_ids = h3ssl->ssl_ids;
239     for (i = 0; i < MAXSSL_IDS; i++) {
240         if (ssl_ids[i].id == UINT64_MAX)
241             continue;
242         printf("are_all_clientid_closed: %llu status %d : %d\n",
243                (unsigned long long) ssl_ids[i].id, ssl_ids[i].status, CLIENTUNIOPEN | CLIENTCLOSED);
244         if (ssl_ids[i].status & CLIENTUNIOPEN) {
245             if (ssl_ids[i].status & CLIENTCLOSED) {
246                 printf("are_all_clientid_closed: %llu closed\n",
247                        (unsigned long long) ssl_ids[i].id);
248                 SSL_free(ssl_ids[i].s);
249                 ssl_ids[i].s = NULL;
250                 ssl_ids[i].id = UINT64_MAX;
251                 continue;
252             }
253             printf("are_all_clientid_closed: %llu open\n", (unsigned long long) ssl_ids[i].id);
254             return 0;
255         }
256     }
257     return 1;
258 }
259 
260 /* free all the ids except listener and connection */
close_all_ids(struct h3ssl * h3ssl)261 static void close_all_ids(struct h3ssl *h3ssl)
262 {
263     struct ssl_id *ssl_ids;
264     int i;
265 
266     ssl_ids = h3ssl->ssl_ids;
267     for (i = 0; i < MAXSSL_IDS; i++) {
268         if (ssl_ids[i].id == UINT64_MAX)
269             continue;
270         SSL_free(ssl_ids[i].s);
271         ssl_ids[i].s = NULL;
272         ssl_ids[i].id = UINT64_MAX;
273     }
274 }
275 
on_recv_header(nghttp3_conn * conn,int64_t stream_id,int32_t token,nghttp3_rcbuf * name,nghttp3_rcbuf * value,uint8_t flags,void * user_data,void * stream_user_data)276 static int on_recv_header(nghttp3_conn *conn, int64_t stream_id, int32_t token,
277                           nghttp3_rcbuf *name, nghttp3_rcbuf *value,
278                           uint8_t flags, void *user_data,
279                           void *stream_user_data)
280 {
281     nghttp3_vec vname, vvalue;
282     struct h3ssl *h3ssl = (struct h3ssl *)user_data;
283 
284     /* Received a single HTTP header. */
285     vname = nghttp3_rcbuf_get_buf(name);
286     vvalue = nghttp3_rcbuf_get_buf(value);
287 
288     fwrite(vname.base, vname.len, 1, stdout);
289     fprintf(stdout, ": ");
290     fwrite(vvalue.base, vvalue.len, 1, stdout);
291     fprintf(stdout, "\n");
292 
293     if (token == NGHTTP3_QPACK_TOKEN__PATH) {
294         int len = (((vvalue.len) < (MAXURL)) ? (vvalue.len) : (MAXURL));
295 
296         memset(h3ssl->url, 0, sizeof(h3ssl->url));
297         if (vvalue.base[0] == '/') {
298             if (vvalue.base[1] == '\0') {
299                 strncpy(h3ssl->url, "index.html", MAXURL);
300             } else {
301                 memcpy(h3ssl->url, vvalue.base + 1, len - 1);
302                 h3ssl->url[len - 1] = '\0';
303             }
304         } else {
305             memcpy(h3ssl->url, vvalue.base, len);
306         }
307     }
308 
309     return 0;
310 }
311 
on_end_headers(nghttp3_conn * conn,int64_t stream_id,int fin,void * user_data,void * stream_user_data)312 static int on_end_headers(nghttp3_conn *conn, int64_t stream_id, int fin,
313                           void *user_data, void *stream_user_data)
314 {
315     struct h3ssl *h3ssl = (struct h3ssl *)user_data;
316 
317     fprintf(stderr, "on_end_headers!\n");
318     h3ssl->end_headers_received = 1;
319     return 0;
320 }
321 
on_recv_data(nghttp3_conn * conn,int64_t stream_id,const uint8_t * data,size_t datalen,void * conn_user_data,void * stream_user_data)322 static int on_recv_data(nghttp3_conn *conn, int64_t stream_id,
323                         const uint8_t *data, size_t datalen,
324                         void *conn_user_data, void *stream_user_data)
325 {
326     fprintf(stderr, "on_recv_data! %ld\n", (unsigned long)datalen);
327     fprintf(stderr, "on_recv_data! %.*s\n", (int)datalen, data);
328     return 0;
329 }
330 
on_end_stream(nghttp3_conn * h3conn,int64_t stream_id,void * conn_user_data,void * stream_user_data)331 static int on_end_stream(nghttp3_conn *h3conn, int64_t stream_id,
332                          void *conn_user_data, void *stream_user_data)
333 {
334     struct h3ssl *h3ssl = (struct h3ssl *)conn_user_data;
335 
336     printf("on_end_stream!\n");
337     h3ssl->done = 1;
338     return 0;
339 }
340 
341 /* Read from the stream and push to the h3conn */
quic_server_read(nghttp3_conn * h3conn,SSL * stream,uint64_t id,struct h3ssl * h3ssl)342 static int quic_server_read(nghttp3_conn *h3conn, SSL *stream, uint64_t id, struct h3ssl *h3ssl)
343 {
344     int ret, r;
345     uint8_t msg2[16000];
346     size_t l = sizeof(msg2);
347 
348     if (!SSL_has_pending(stream))
349         return 0; /* Nothing to read */
350 
351     ret = SSL_read(stream, msg2, l);
352     if (ret <= 0) {
353         fprintf(stderr, "SSL_read %d on %llu failed\n",
354                 SSL_get_error(stream, ret),
355                 (unsigned long long) id);
356         switch (SSL_get_error(stream, ret)) {
357         case SSL_ERROR_WANT_READ:
358             return 0;
359         case SSL_ERROR_ZERO_RETURN:
360             return 1;
361         default:
362             ERR_print_errors_fp(stderr);
363             return -1;
364         }
365         return -1;
366     }
367 
368     /* XXX: work around nghttp3_conn_read_stream returning  -607 on stream 2 */
369     if (!h3ssl->received_from_two && id != 2) {
370         r = nghttp3_conn_read_stream(h3conn, id, msg2, ret, 0);
371     } else {
372         r = ret; /* ignore it for the moment ... */
373     }
374 
375     printf("nghttp3_conn_read_stream used %d of %d on %llu\n", r,
376            ret, (unsigned long long) id);
377     if (r != ret) {
378         /* chrome returns -607 on stream 2 */
379         if (!nghttp3_err_is_fatal(r)) {
380             printf("nghttp3_conn_read_stream used %d of %d (not fatal) on %llu\n", r,
381                    ret, (unsigned long long) id);
382             if (id == 2)
383                 h3ssl->received_from_two = 1;
384             return 1;
385         }
386         return -1;
387     }
388     return 1;
389 }
390 
391 /*
392  * creates the control stream, the encoding and decoding streams.
393  * nghttp3_conn_bind_control_stream() is for the control stream.
394  */
quic_server_h3streams(nghttp3_conn * h3conn,struct h3ssl * h3ssl)395 static int quic_server_h3streams(nghttp3_conn *h3conn, struct h3ssl *h3ssl)
396 {
397     SSL *rstream = NULL;
398     SSL *pstream = NULL;
399     SSL *cstream = NULL;
400     SSL *conn;
401     uint64_t r_streamid, p_streamid, c_streamid;
402 
403     conn = get_ids_connection(h3ssl);
404     if (conn == NULL) {
405         fprintf(stderr, "quic_server_h3streams no connection\n");
406         fflush(stderr);
407         return -1;
408     }
409     rstream = SSL_new_stream(conn, SSL_STREAM_FLAG_UNI);
410     if (rstream != NULL) {
411         printf("=> Opened on %llu\n",
412                (unsigned long long)SSL_get_stream_id(rstream));
413     } else {
414         fprintf(stderr, "=> Stream == NULL!\n");
415         goto err;
416     }
417     pstream = SSL_new_stream(conn, SSL_STREAM_FLAG_UNI);
418     if (pstream != NULL) {
419         printf("=> Opened on %llu\n",
420                (unsigned long long)SSL_get_stream_id(pstream));
421     } else {
422         fprintf(stderr, "=> Stream == NULL!\n");
423         goto err;
424     }
425     cstream = SSL_new_stream(conn, SSL_STREAM_FLAG_UNI);
426     if (cstream != NULL) {
427         fprintf(stderr, "=> Opened on %llu\n",
428                 (unsigned long long)SSL_get_stream_id(cstream));
429         fflush(stderr);
430     } else {
431         fprintf(stderr, "=> Stream == NULL!\n");
432         goto err;
433     }
434     r_streamid = SSL_get_stream_id(rstream);
435     p_streamid = SSL_get_stream_id(pstream);
436     c_streamid = SSL_get_stream_id(cstream);
437     if (nghttp3_conn_bind_qpack_streams(h3conn, p_streamid, r_streamid)) {
438         fprintf(stderr, "nghttp3_conn_bind_qpack_streams failed!\n");
439         goto err;
440     }
441     if (nghttp3_conn_bind_control_stream(h3conn, c_streamid)) {
442         fprintf(stderr, "nghttp3_conn_bind_qpack_streams failed!\n");
443         goto err;
444     }
445     printf("control: %llu enc %llu dec %llu\n",
446            (unsigned long long)c_streamid,
447            (unsigned long long)p_streamid,
448            (unsigned long long)r_streamid);
449     add_id(SSL_get_stream_id(rstream), rstream, h3ssl);
450     add_id(SSL_get_stream_id(pstream), pstream, h3ssl);
451     add_id(SSL_get_stream_id(cstream), cstream, h3ssl);
452 
453     return 0;
454 err:
455     fflush(stderr);
456     SSL_free(rstream);
457     SSL_free(pstream);
458     SSL_free(cstream);
459     return -1;
460 }
461 
462 /* Try to read from the streams we have */
read_from_ssl_ids(nghttp3_conn ** curh3conn,struct h3ssl * h3ssl)463 static int read_from_ssl_ids(nghttp3_conn **curh3conn, struct h3ssl *h3ssl)
464 {
465     int hassomething = 0, i;
466     struct ssl_id *ssl_ids = h3ssl->ssl_ids;
467     SSL_POLL_ITEM items[MAXSSL_IDS] = {0}, *item = items;
468     static const struct timeval nz_timeout = {0, 0};
469     size_t result_count = SIZE_MAX;
470     int numitem = 0, ret;
471     uint64_t processed_event = 0;
472     int has_ids_to_remove = 0;
473     nghttp3_conn *h3conn = *curh3conn;
474 
475     /*
476      * Process all the streams
477      * the first one is the connection if we get something here is a new stream
478      */
479     for (i = 0; i < MAXSSL_IDS; i++) {
480         if (ssl_ids[i].s != NULL) {
481             item->desc = SSL_as_poll_descriptor(ssl_ids[i].s);
482             item->events = UINT64_MAX;  /* TODO adjust to the event we need process */
483             item->revents = UINT64_MAX; /* TODO adjust to the event we need process */
484             numitem++;
485             item++;
486         }
487     }
488 
489     /*
490      * SSL_POLL_FLAG_NO_HANDLE_EVENTS would require to use:
491      * SSL_get_event_timeout on the connection stream
492      * select/wait using the timeout value (which could be no wait time)
493      * SSL_handle_events
494      * SSL_poll
495      * for the moment we let SSL_poll to performs ticking internally
496      * on an automatic basis.
497      */
498     ret = SSL_poll(items, numitem, sizeof(SSL_POLL_ITEM), &nz_timeout,
499                    SSL_POLL_FLAG_NO_HANDLE_EVENTS, &result_count);
500     if (!ret) {
501         fprintf(stderr, "SSL_poll failed\n");
502         printf("SSL_poll failed\n");
503         return -1; /* something is wrong */
504     }
505     printf("read_from_ssl_ids %ld events\n", (unsigned long)result_count);
506     if (result_count == 0) {
507         /* Timeout may be something somewhere */
508         return 0;
509     }
510 
511     /* reset the states */
512     h3ssl->new_conn = 0;
513     h3ssl->restart = 0;
514     h3ssl->done = 0;
515 
516     /* Process all the item we have polled */
517     for (i = 0, item = items; i < numitem; i++, item++) {
518         SSL *s;
519 
520         if (item->revents == SSL_POLL_EVENT_NONE)
521             continue;
522         processed_event = 0;
523         /* get the stream */
524         s = item->desc.value.ssl;
525 
526         /* New connection */
527         if (item->revents & SSL_POLL_EVENT_IC) {
528             SSL *conn = SSL_accept_connection(item->desc.value.ssl, 0);
529             SSL *oldconn;
530 
531             printf("SSL_accept_connection\n");
532             if (conn == NULL) {
533                 fprintf(stderr, "error while accepting connection\n");
534                 ret = -1;
535                 goto err;
536             }
537 
538             /* the previous might be still there */
539             oldconn = get_ids_connection(h3ssl);
540             if (oldconn != NULL) {
541                 /* XXX we support only one connection for the moment */
542                 printf("SSL_accept_connection closing previous\n");
543                 SSL_free(oldconn);
544                 replace_ids_connection(h3ssl, oldconn, conn);
545                 reuse_h3ssl(h3ssl);
546                 close_all_ids(h3ssl);
547                 h3ssl->id_bidi = UINT64_MAX;
548                 h3ssl->has_uni = 0;
549             } else {
550                 printf("SSL_accept_connection first connection\n");
551                 add_ids_connection(h3ssl, conn);
552             }
553             h3ssl->new_conn = 1;
554             /* create the new h3conn */
555             nghttp3_conn_del(*curh3conn);
556             nghttp3_settings_default(&settings);
557             if (nghttp3_conn_server_new(curh3conn, &callbacks, &settings, mem,
558                                         h3ssl)) {
559                 fprintf(stderr, "nghttp3_conn_client_new failed!\n");
560                 exit(1);
561             }
562             h3conn = *curh3conn;
563             hassomething++;
564 
565             if (!SSL_set_incoming_stream_policy(conn,
566                                                 SSL_INCOMING_STREAM_POLICY_ACCEPT, 0)) {
567                 fprintf(stderr, "error while setting inccoming stream policy\n");
568                 ret = -1;
569                 goto err;
570             }
571 
572             printf("SSL_accept_connection\n");
573             processed_event = processed_event | SSL_POLL_EVENT_IC;
574         }
575         /* SSL_accept_stream if SSL_POLL_EVENT_ISB or SSL_POLL_EVENT_ISU */
576         if ((item->revents & SSL_POLL_EVENT_ISB) ||
577             (item->revents & SSL_POLL_EVENT_ISU)) {
578             SSL *stream = SSL_accept_stream(item->desc.value.ssl, 0);
579             uint64_t new_id;
580             int r;
581 
582             if (stream == NULL) {
583                 ret = -1;
584                 goto err;
585             }
586             new_id = SSL_get_stream_id(stream);
587             printf("=> Received connection on %lld %d\n", (unsigned long long) new_id,
588                    SSL_get_stream_type(stream));
589             add_id(new_id, stream, h3ssl);
590             if (h3ssl->close_wait) {
591                 printf("in close_wait so we will have a new request\n");
592                 reuse_h3ssl(h3ssl);
593                 h3ssl->restart = 1; /* Checked in wait_close loop */
594             }
595             if (SSL_get_stream_type(stream) == SSL_STREAM_TYPE_BIDI) {
596                 /* bidi that is the id  where we have to send the response */
597                 if (h3ssl->id_bidi != UINT64_MAX) {
598                     set_id_status(h3ssl->id_bidi, TOBEREMOVED, h3ssl);
599                     has_ids_to_remove++;
600                 }
601                 h3ssl->id_bidi = new_id;
602                 reuse_h3ssl(h3ssl);
603                 h3ssl->restart = 1;
604             } else {
605                 set_id_status(new_id, CLIENTUNIOPEN, h3ssl);
606             }
607 
608             r = quic_server_read(h3conn, stream, new_id, h3ssl);
609             if (r == -1) {
610                 ret = -1;
611                 goto err;
612             }
613             if (r == 1)
614                 hassomething++;
615 
616             if (item->revents & SSL_POLL_EVENT_ISB)
617                 processed_event = processed_event | SSL_POLL_EVENT_ISB;
618             if (item->revents & SSL_POLL_EVENT_ISU)
619                 processed_event = processed_event | SSL_POLL_EVENT_ISU;
620         }
621         if (item->revents & SSL_POLL_EVENT_OSB) {
622             /* Create new streams when allowed */
623             /* at least one bidi */
624             processed_event = processed_event | SSL_POLL_EVENT_OSB;
625             printf("Create bidi?\n");
626         }
627         if (item->revents & SSL_POLL_EVENT_OSU) {
628             /* at least one uni */
629             /* we have 4 streams from the client 2, 6 , 10 and 0 */
630             /* need 3 streams to the client */
631             printf("Create uni?\n");
632             processed_event = processed_event | SSL_POLL_EVENT_OSU;
633             if (!h3ssl->has_uni) {
634                 printf("Create uni\n");
635                 ret = quic_server_h3streams(h3conn, h3ssl);
636                 if (ret == -1) {
637                     fprintf(stderr, "quic_server_h3streams failed!\n");
638                     goto err;
639                 }
640                 h3ssl->has_uni = 1;
641                 hassomething++;
642             }
643         }
644         if (item->revents & SSL_POLL_EVENT_EC) {
645             /* the connection begins terminating */
646             printf("Connection terminating\n");
647             printf("Connection terminating restart %d\n", h3ssl->restart);
648             if (!h3ssl->close_done) {
649                 h3ssl->close_done = 1;
650             } else {
651                 h3ssl->done = 1;
652             }
653             hassomething++;
654             processed_event = processed_event | SSL_POLL_EVENT_EC;
655         }
656         if (item->revents & SSL_POLL_EVENT_ECD) {
657             /* the connection is terminated */
658             printf("Connection terminated\n");
659             h3ssl->done = 1;
660             hassomething++;
661             processed_event = processed_event | SSL_POLL_EVENT_ECD;
662         }
663 
664         if (item->revents & SSL_POLL_EVENT_R) {
665             /* try to read */
666             uint64_t id = UINT64_MAX;
667             int r;
668 
669             /* get the id, well the connection has no id... */
670             id = SSL_get_stream_id(item->desc.value.ssl);
671             printf("revent READ on %llu\n", (unsigned long long)id);
672             r = quic_server_read(h3conn, s, id, h3ssl);
673             if (r == 0) {
674                 uint8_t msg[1];
675                 size_t l = sizeof(msg);
676 
677                 /* check that the other side is closed */
678                 r = SSL_read(s, msg, l);
679                 printf("SSL_read tells %d\n", r);
680                 if (r > 0) {
681                     ret = -1;
682                     goto err;
683                 }
684                 r = SSL_get_error(s, r);
685                 if (r != SSL_ERROR_ZERO_RETURN) {
686                     ret = -1;
687                     goto err;
688                 }
689                 set_id_status(id, TOBEREMOVED, h3ssl);
690                 has_ids_to_remove++;
691                 continue;
692             }
693             if (r == -1) {
694                 ret = -1;
695                 goto err;
696             }
697             hassomething++;
698             processed_event = processed_event | SSL_POLL_EVENT_R;
699         }
700         if (item->revents & SSL_POLL_EVENT_ER) {
701             /* mark it closed */
702             uint64_t id = UINT64_MAX;
703             int status;
704 
705             id = SSL_get_stream_id(item->desc.value.ssl);
706             status = get_id_status(id, h3ssl);
707 
708             printf("revent exception READ on %llu\n", (unsigned long long)id);
709             if (status & CLIENTUNIOPEN) {
710                 set_id_status(id, CLIENTCLOSED, h3ssl);
711                 hassomething++;
712             }
713             processed_event = processed_event | SSL_POLL_EVENT_ER;
714         }
715         if (item->revents & SSL_POLL_EVENT_W) {
716             /* we ignore those for the moment */
717             processed_event = processed_event | SSL_POLL_EVENT_W;
718         }
719         if (item->revents & SSL_POLL_EVENT_EW) {
720             /* write part received a STOP_SENDING */
721             uint64_t id = UINT64_MAX;
722             int status;
723 
724             id = SSL_get_stream_id(item->desc.value.ssl);
725             status = get_id_status(id, h3ssl);
726 
727             if (status & SERVERCLOSED) {
728                 printf("both sides closed on  %llu\n", (unsigned long long)id);
729                 set_id_status(id, TOBEREMOVED, h3ssl);
730                 has_ids_to_remove++;
731                 hassomething++;
732             }
733             processed_event = processed_event | SSL_POLL_EVENT_EW;
734         }
735         if (item->revents != processed_event) {
736             /* Figure out ??? */
737             uint64_t id = UINT64_MAX;
738 
739             id = SSL_get_stream_id(item->desc.value.ssl);
740             printf("revent %llu (%d) on %llu NOT PROCESSED!\n",
741                    (unsigned long long)item->revents, SSL_POLL_EVENT_W,
742                    (unsigned long long)id);
743         }
744     }
745     ret = hassomething;
746 err:
747     if (has_ids_to_remove)
748         remove_marked_ids(h3ssl);
749     return ret;
750 }
751 
handle_events_from_ids(struct h3ssl * h3ssl)752 static void handle_events_from_ids(struct h3ssl *h3ssl)
753 {
754     struct ssl_id *ssl_ids = h3ssl->ssl_ids;
755     int i;
756 
757     ssl_ids = h3ssl->ssl_ids;
758     for (i = 0; i < MAXSSL_IDS; i++) {
759         if (ssl_ids[i].s != NULL &&
760             (ssl_ids[i].status & ISCONNECTION || ssl_ids[i].status & ISLISTENER)) {
761             if (SSL_handle_events(ssl_ids[i].s))
762                 ERR_print_errors_fp(stderr);
763         }
764     }
765 }
766 
get_file_length(struct h3ssl * h3ssl)767 static size_t get_file_length(struct h3ssl *h3ssl)
768 {
769     char filename[PATH_MAX];
770     struct stat st;
771 
772     memset(filename, 0, PATH_MAX);
773     if (h3ssl->fileprefix != NULL)
774         strcat(filename, h3ssl->fileprefix);
775     strcat(filename, h3ssl->url);
776 
777     if (strcmp(h3ssl->url, "big") == 0) {
778         printf("big!!!\n");
779         return (size_t)INT_MAX;
780     }
781     if (stat(filename, &st) == 0) {
782         /* Only process regular files */
783         if (S_ISREG(st.st_mode)) {
784             printf("get_file_length %s %lld\n", filename, (unsigned long long) st.st_size);
785             return (size_t)st.st_size;
786         }
787     }
788     printf("Can't get_file_length %s\n", filename);
789     return 0;
790 }
791 
get_file_data(struct h3ssl * h3ssl)792 static char *get_file_data(struct h3ssl *h3ssl)
793 {
794     char filename[PATH_MAX];
795     size_t size = get_file_length(h3ssl);
796     char *res;
797     int fd;
798 
799     if (size == 0)
800         return NULL;
801 
802     memset(filename, 0, PATH_MAX);
803     if (h3ssl->fileprefix != NULL)
804         strcat(filename, h3ssl->fileprefix);
805     strcat(filename, h3ssl->url);
806 
807     res = malloc(size+1);
808     res[size] = '\0';
809     fd = open(filename, O_RDONLY);
810     if (read(fd, res, size) == -1) {
811         close(fd);
812         free(res);
813         return NULL;
814     }
815     close(fd);
816     printf("read from %s : %zu\n", filename, size);
817     return res;
818 }
819 
step_read_data(nghttp3_conn * conn,int64_t stream_id,nghttp3_vec * vec,size_t veccnt,uint32_t * pflags,void * user_data,void * stream_user_data)820 static nghttp3_ssize step_read_data(nghttp3_conn *conn, int64_t stream_id,
821                                     nghttp3_vec *vec, size_t veccnt,
822                                     uint32_t *pflags, void *user_data,
823                                     void *stream_user_data)
824 {
825     struct h3ssl *h3ssl = (struct h3ssl *)user_data;
826 
827     if (h3ssl->datadone) {
828         *pflags = NGHTTP3_DATA_FLAG_EOF;
829         return 0;
830     }
831     /* send the data */
832     printf("step_read_data for %s %zu\n", h3ssl->url, h3ssl->ldata);
833     if (h3ssl->ldata <= 4096) {
834         vec[0].base = &(h3ssl->ptr_data[h3ssl->offset_data]);
835         vec[0].len = h3ssl->ldata;
836         h3ssl->datadone++;
837         *pflags = NGHTTP3_DATA_FLAG_EOF;
838     } else {
839         vec[0].base = &(h3ssl->ptr_data[h3ssl->offset_data]);
840         vec[0].len = 4096;
841         if (h3ssl->ldata == INT_MAX) {
842             printf("big = endless!\n");
843         } else {
844             h3ssl->offset_data = h3ssl->offset_data + 4096;
845             h3ssl->ldata = h3ssl->ldata - 4096;
846         }
847     }
848 
849     return 1;
850 }
851 
quic_server_write(struct h3ssl * h3ssl,uint64_t streamid,uint8_t * buff,size_t len,uint64_t flags,size_t * written)852 static int quic_server_write(struct h3ssl *h3ssl, uint64_t streamid,
853                              uint8_t *buff, size_t len, uint64_t flags,
854                              size_t *written)
855 {
856     struct ssl_id *ssl_ids;
857     int i;
858 
859     ssl_ids = h3ssl->ssl_ids;
860     for (i = 0; i < MAXSSL_IDS; i++) {
861         if (ssl_ids[i].id == streamid) {
862             if (!SSL_write_ex2(ssl_ids[i].s, buff, len, flags, written) ||
863                 *written != len) {
864                 fprintf(stderr, "couldn't write on connection\n");
865                 ERR_print_errors_fp(stderr);
866                 return 0;
867             }
868             printf("written %lld on %lld flags %lld\n", (unsigned long long)len,
869                    (unsigned long long)streamid, (unsigned long long)flags);
870             return 1;
871         }
872     }
873     printf("quic_server_write %lld on %lld (NOT FOUND!)\n", (unsigned long long)len,
874            (unsigned long long)streamid);
875     return 0;
876 }
877 
878 #define OSSL_NELEM(x) (sizeof(x) / sizeof((x)[0]))
879 
880 /*
881  * This is a basic demo of QUIC server functionality in which one connection at
882  * a time is accepted in a blocking loop.
883  */
884 
885 /* ALPN string for TLS handshake. We pretent h3-29 and h3 */
886 static const unsigned char alpn_ossltest[] = { 5,   'h', '3', '-', '2',
887                                                '9', 2,   'h', '3' };
888 
889 /*
890  * This callback validates and negotiates the desired ALPN on the server side.
891  */
select_alpn(SSL * ssl,const unsigned char ** out,unsigned char * out_len,const unsigned char * in,unsigned int in_len,void * arg)892 static int select_alpn(SSL *ssl, const unsigned char **out,
893                        unsigned char *out_len, const unsigned char *in,
894                        unsigned int in_len, void *arg)
895 {
896     if (SSL_select_next_proto((unsigned char **)out, out_len, alpn_ossltest,
897                               sizeof(alpn_ossltest), in,
898                               in_len) != OPENSSL_NPN_NEGOTIATED)
899         return SSL_TLSEXT_ERR_ALERT_FATAL;
900 
901     return SSL_TLSEXT_ERR_OK;
902 }
903 
904 /* Create SSL_CTX. */
create_ctx(const char * cert_path,const char * key_path)905 static SSL_CTX *create_ctx(const char *cert_path, const char *key_path)
906 {
907     SSL_CTX *ctx;
908 
909     ctx = SSL_CTX_new(OSSL_QUIC_server_method());
910     if (ctx == NULL)
911         goto err;
912 
913     /* Load certificate and corresponding private key. */
914     if (SSL_CTX_use_certificate_chain_file(ctx, cert_path) <= 0) {
915         fprintf(stderr, "couldn't load certificate file: %s\n", cert_path);
916         goto err;
917     }
918 
919     if (SSL_CTX_use_PrivateKey_file(ctx, key_path, SSL_FILETYPE_PEM) <= 0) {
920         fprintf(stderr, "couldn't load key file: %s\n", key_path);
921         goto err;
922     }
923 
924     if (!SSL_CTX_check_private_key(ctx)) {
925         fprintf(stderr, "private key check failed\n");
926         goto err;
927     }
928 
929     /* Setup ALPN negotiation callback. */
930     SSL_CTX_set_alpn_select_cb(ctx, select_alpn, NULL);
931     return ctx;
932 
933 err:
934     SSL_CTX_free(ctx);
935     return NULL;
936 }
937 
938 /* Create UDP socket using given port. */
create_socket(uint16_t port)939 static int create_socket(uint16_t port)
940 {
941     int fd = -1;
942     struct sockaddr_in sa = {0};
943 
944     if ((fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) {
945         fprintf(stderr, "cannot create socket");
946         goto err;
947     }
948 
949     sa.sin_family = AF_INET;
950     sa.sin_port = htons(port);
951 
952     if (bind(fd, (const struct sockaddr *)&sa, sizeof(sa)) < 0) {
953         fprintf(stderr, "cannot bind to %u\n", port);
954         goto err;
955     }
956 
957     return fd;
958 
959 err:
960     if (fd >= 0)
961         BIO_closesocket(fd);
962 
963     return -1;
964 }
965 
966 /* Copied from demos/guide/quic-server-non-block.c */
967 /**
968  * @brief Waits for activity on the SSL socket, either for reading or writing.
969  *
970  * This function monitors the underlying file descriptor of the given SSL
971  * connection to determine when it is ready for reading or writing, or both.
972  * It uses the select function to wait until the socket is either readable
973  * or writable, depending on what the SSL connection requires.
974  *
975  * @param ssl A pointer to the SSL object representing the connection.
976  *
977  * @note This function blocks until there is activity on the socket. In a real
978  * application, you might want to perform other tasks while waiting, such as
979  * updating a GUI or handling other connections.
980  *
981  * @note This function uses select for simplicity and portability. Depending
982  * on your application's requirements, you might consider using other
983  * mechanisms like poll or epoll for handling multiple file descriptors.
984  */
wait_for_activity(SSL * ssl)985 static int wait_for_activity(SSL *ssl)
986 {
987     int sock, isinfinite;
988     fd_set read_fd, write_fd;
989     struct timeval tv;
990     struct timeval *tvp = NULL;
991 
992     /* Get hold of the underlying file descriptor for the socket */
993     if ((sock = SSL_get_fd(ssl)) == -1) {
994         fprintf(stderr, "Unable to get file descriptor");
995         return -1;
996     }
997 
998     /* Initialize the fd_set structure */
999     FD_ZERO(&read_fd);
1000     FD_ZERO(&write_fd);
1001 
1002     /*
1003      * Determine if we would like to write to the socket, read from it, or both.
1004      */
1005     if (SSL_net_write_desired(ssl))
1006         FD_SET(sock, &write_fd);
1007     if (SSL_net_read_desired(ssl))
1008         FD_SET(sock, &read_fd);
1009 
1010     /* Add the socket file descriptor to the fd_set */
1011     FD_SET(sock, &read_fd);
1012 
1013     /*
1014      * Find out when OpenSSL would next like to be called, regardless of
1015      * whether the state of the underlying socket has changed or not.
1016      */
1017     if (SSL_get_event_timeout(ssl, &tv, &isinfinite) && !isinfinite)
1018         tvp = &tv;
1019 
1020     /*
1021      * Wait until the socket is writeable or readable. We use select here
1022      * for the sake of simplicity and portability, but you could equally use
1023      * poll/epoll or similar functions
1024      *
1025      * NOTE: For the purposes of this demonstration code this effectively
1026      * makes this demo block until it has something more useful to do. In a
1027      * real application you probably want to go and do other work here (e.g.
1028      * update a GUI, or service other connections).
1029      *
1030      * Let's say for example that you want to update the progress counter on
1031      * a GUI every 100ms. One way to do that would be to use the timeout in
1032      * the last parameter to "select" below. If the tvp value is greater
1033      * than 100ms then use 100ms instead. Then, when select returns, you
1034      * check if it did so because of activity on the file descriptors or
1035      * because of the timeout. If the 100ms GUI timeout has expired but the
1036      * tvp timeout has not then go and update the GUI and then restart the
1037      * "select" (with updated timeouts).
1038      */
1039 
1040     return (select(sock + 1, &read_fd, &write_fd, NULL, tvp));
1041 }
1042 
1043 /* Main loop for server to accept QUIC connections. */
run_quic_server(SSL_CTX * ctx,int fd)1044 static int run_quic_server(SSL_CTX *ctx, int fd)
1045 {
1046     int ok = 0;
1047     int hassomething = 0;
1048     SSL *listener = NULL;
1049     nghttp3_conn *h3conn = NULL;
1050     struct h3ssl h3ssl;
1051     SSL *ssl;
1052     char *fileprefix = getenv("FILEPREFIX");
1053 
1054     /* Create a new QUIC listener. */
1055     if ((listener = SSL_new_listener(ctx, 0)) == NULL)
1056         goto err;
1057 
1058     /* Provide the listener with our UDP socket. */
1059     if (!SSL_set_fd(listener, fd))
1060         goto err;
1061 
1062     /* Begin listening. */
1063     if (!SSL_listen(listener))
1064         goto err;
1065 
1066     /*
1067      * Listeners, and other QUIC objects, default to operating in blocking mode.
1068      * The configured behaviour is inherited by child objects.
1069      * Make sure we won't block as we use select().
1070      */
1071     if (!SSL_set_blocking_mode(listener, 0))
1072         goto err;
1073 
1074     /* Setup callbacks. */
1075     callbacks.recv_header = on_recv_header;
1076     callbacks.end_headers = on_end_headers;
1077     callbacks.recv_data = on_recv_data;
1078     callbacks.end_stream = on_end_stream;
1079 
1080     /* mem default */
1081     mem = nghttp3_mem_default();
1082 
1083     for (;;) {
1084         nghttp3_nv resp[10];
1085         size_t num_nv;
1086         nghttp3_data_reader dr;
1087         int ret;
1088         int numtimeout;
1089         char slength[22];
1090         int hasnothing;
1091 
1092         init_ids(&h3ssl);
1093         h3ssl.fileprefix = fileprefix;
1094         printf("listener: %p\n", (void *)listener);
1095         add_ids_listener(listener, &h3ssl);
1096 
1097         if (!hassomething) {
1098             printf("waiting on socket\n");
1099             fflush(stdout);
1100             ret = wait_for_activity(listener);
1101             if (ret == -1) {
1102                 fprintf(stderr, "wait_for_activity failed!\n");
1103                 goto err;
1104             }
1105         }
1106         /*
1107          * Service the connection. In a real application this would be done
1108          * concurrently. In this demonstration program a single connection is
1109          * accepted and serviced at a time.
1110          */
1111     newconn:
1112 
1113         printf("process_server starting...\n");
1114         fflush(stdout);
1115 
1116         /* wait until we have received the headers */
1117     restart:
1118         numtimeout = 0;
1119         num_nv = 0;
1120         while (!h3ssl.end_headers_received) {
1121             if (!hassomething) {
1122                 if (wait_for_activity(listener) == 0) {
1123                     printf("waiting for end_headers_received timeout %d\n", numtimeout);
1124                     numtimeout++;
1125                     if (numtimeout == 25)
1126                         goto err;
1127                 }
1128                 handle_events_from_ids(&h3ssl);
1129             }
1130             hassomething = read_from_ssl_ids(&h3conn, &h3ssl);
1131             if (hassomething == -1) {
1132                 fprintf(stderr, "read_from_ssl_ids hassomething failed\n");
1133                 goto err;
1134             } else if (hassomething == 0) {
1135                 printf("read_from_ssl_ids hassomething nothing...\n");
1136             } else {
1137                 numtimeout = 0;
1138                 printf("read_from_ssl_ids hassomething %d...\n", hassomething);
1139                 if (h3ssl.close_done) {
1140                     /* Other side has closed */
1141                     break;
1142                 }
1143                 h3ssl.restart = 0;
1144             }
1145         }
1146         if (h3ssl.close_done) {
1147             printf("Other side close without request\n");
1148             goto wait_close;
1149         }
1150         printf("end_headers_received!!!\n");
1151         if (!h3ssl.has_uni) {
1152             /* time to create those otherwise we can't push anything to the client */
1153             printf("Create uni\n");
1154             if (quic_server_h3streams(h3conn, &h3ssl) == -1) {
1155                 fprintf(stderr, "quic_server_h3streams failed!\n");
1156                 goto err;
1157             }
1158             h3ssl.has_uni = 1;
1159         }
1160 
1161         /* we have receive the request build the response and send it */
1162         /* XXX add  MAKE_NV("connection", "close"), to resp[] and recheck */
1163         make_nv(&resp[num_nv++], ":status", "200");
1164         h3ssl.ldata = get_file_length(&h3ssl);
1165         if (h3ssl.ldata == 0) {
1166             /* We don't find the file: use default test string */
1167             h3ssl.ptr_data = nulldata;
1168             h3ssl.ldata = nulldata_sz;
1169             sprintf(slength, "%zu", h3ssl.ldata);
1170             /* content-type: text/html */
1171             make_nv(&resp[num_nv++], "content-type", "text/html");
1172         } else if (h3ssl.ldata == INT_MAX) {
1173             /* endless file for tests */
1174             sprintf(slength, "%zu", h3ssl.ldata);
1175             h3ssl.ptr_data = (uint8_t *) malloc(4096);
1176             memset(h3ssl.ptr_data, 'A', 4096);
1177         } else {
1178             /* normal file we have opened */
1179             sprintf(slength, "%zu", h3ssl.ldata);
1180             h3ssl.ptr_data = (uint8_t *) get_file_data(&h3ssl);
1181             if (h3ssl.ptr_data == NULL)
1182                 abort();
1183             printf("before nghttp3_conn_submit_response on %llu for %s ...\n",
1184                    (unsigned long long) h3ssl.id_bidi, h3ssl.url);
1185             if (strstr(h3ssl.url, ".png"))
1186                 make_nv(&resp[num_nv++], "content-type", "image/png");
1187             else if (strstr(h3ssl.url, ".ico"))
1188                 make_nv(&resp[num_nv++], "content-type", "image/vnd.microsoft.icon");
1189             else if (strstr(h3ssl.url, ".htm"))
1190                 make_nv(&resp[num_nv++], "content-type", "text/html");
1191             else
1192                 make_nv(&resp[num_nv++], "content-type", "application/octet-stream");
1193             make_nv(&resp[num_nv++], "content-length", slength);
1194         }
1195 
1196         dr.read_data = step_read_data;
1197         if (nghttp3_conn_submit_response(h3conn, h3ssl.id_bidi, resp, num_nv, &dr)) {
1198             fprintf(stderr, "nghttp3_conn_submit_response failed!\n");
1199             goto err;
1200         }
1201         printf("nghttp3_conn_submit_response on %llu...\n", (unsigned long long) h3ssl.id_bidi);
1202         for (;;) {
1203             nghttp3_vec vec[256];
1204             nghttp3_ssize sveccnt;
1205             int fin, i;
1206             int64_t streamid;
1207 
1208             sveccnt = nghttp3_conn_writev_stream(h3conn, &streamid, &fin, vec,
1209                                                  nghttp3_arraylen(vec));
1210             if (sveccnt <= 0) {
1211                 printf("nghttp3_conn_writev_stream done: %ld stream: %llu fin %d\n",
1212                        (long int)sveccnt,
1213                        (unsigned long long)streamid,
1214                        fin);
1215                 if (streamid != -1 && fin) {
1216                     printf("Sending end data on %llu fin %d\n",
1217                            (unsigned long long) streamid, fin);
1218                     nghttp3_conn_add_write_offset(h3conn, streamid, 0);
1219                     continue;
1220                 }
1221                 if (!h3ssl.datadone)
1222                     goto err;
1223                 else
1224                     break; /* Done */
1225             }
1226             printf("nghttp3_conn_writev_stream: %ld fin: %d\n", (long int)sveccnt, fin);
1227             for (i = 0; i < sveccnt; i++) {
1228                 size_t numbytes = vec[i].len;
1229                 int flagwrite = 0;
1230 
1231                 printf("quic_server_write on %llu for %ld\n",
1232                        (unsigned long long)streamid, (unsigned long)vec[i].len);
1233                 if (fin && i == sveccnt - 1)
1234                     flagwrite = SSL_WRITE_FLAG_CONCLUDE;
1235                 if (!quic_server_write(&h3ssl, streamid, vec[i].base,
1236                                        vec[i].len, flagwrite, &numbytes)) {
1237                     fprintf(stderr, "quic_server_write failed!\n");
1238                     goto err;
1239                 }
1240             }
1241             if (nghttp3_conn_add_write_offset(
1242                                               h3conn, streamid,
1243                                               (size_t)nghttp3_vec_len(vec, (size_t)sveccnt))) {
1244                 fprintf(stderr, "nghttp3_conn_add_write_offset failed!\n");
1245                 goto err;
1246             }
1247         }
1248         printf("nghttp3_conn_submit_response DONE!!!\n");
1249 
1250         if (h3ssl.datadone) {
1251             /*
1252              * All the data was sent.
1253              * close stream zero
1254              */
1255             if (!h3ssl.close_done) {
1256                 set_id_status(h3ssl.id_bidi, SERVERCLOSED, &h3ssl);
1257                 h3ssl.close_wait = 1;
1258             }
1259         } else {
1260             printf("nghttp3_conn_submit_response still not finished\n");
1261         }
1262 
1263         /* wait until closed */
1264     wait_close:
1265         hasnothing = 0;
1266         for (;;) {
1267 
1268             if (!hasnothing) {
1269                 SSL *newssl = get_ids_connection(&h3ssl);
1270 
1271                 printf("hasnothing nothing WAIT %d!!!\n", h3ssl.close_done);
1272                 if (newssl == NULL)
1273                     newssl = listener;
1274                 ret = wait_for_activity(newssl);
1275                 if (ret == -1)
1276                     goto err;
1277                 if (ret == 0)
1278                     printf("hasnothing timeout\n");
1279                 /* we have something or a timeout */
1280                 handle_events_from_ids(&h3ssl);
1281             }
1282             hasnothing = read_from_ssl_ids(&h3conn, &h3ssl);
1283             if (hasnothing == -1) {
1284                 printf("hasnothing failed\n");
1285                 break;
1286                 /* goto err; well in fact not */
1287             } else if (hasnothing == 0) {
1288                 printf("hasnothing nothing\n");
1289                 continue;
1290             } else {
1291                 printf("hasnothing something\n");
1292                 if (h3ssl.done) {
1293                     printf("hasnothing something... DONE\n");
1294                     /* we might already have the next connection to accept */
1295                     hassomething = 1;
1296                     break;
1297                 }
1298                 if (h3ssl.new_conn) {
1299                     printf("hasnothing something... NEW CONN\n");
1300                     h3ssl.new_conn = 0;
1301                     goto newconn;
1302                 }
1303                 if (h3ssl.restart) {
1304                     printf("hasnothing something... RESTART\n");
1305                     h3ssl.restart = 0;
1306                     goto restart;
1307                 }
1308                 if (are_all_clientid_closed(&h3ssl)) {
1309                     printf("hasnothing something... DONE other side closed\n");
1310                     /* there might 2 or 3 message we will ignore */
1311                     hassomething = 0;
1312                     break;
1313                 }
1314             }
1315         }
1316 
1317         /*
1318          * Free the streams, then loop again, accepting another connection.
1319          */
1320         close_all_ids(&h3ssl);
1321         ssl = get_ids_connection(&h3ssl);
1322         if (ssl != NULL) {
1323             SSL_free(ssl);
1324             replace_ids_connection(&h3ssl, ssl, NULL);
1325         }
1326         hassomething = 0;
1327     }
1328 
1329     ok = 1;
1330 err:
1331     if (!ok)
1332         ERR_print_errors_fp(stderr);
1333 
1334     SSL_free(listener);
1335     return ok;
1336 }
1337 
1338 /*
1339  * demo server... just return a 20 bytes ascii string as response for any
1340  * request single h3 connection and single threaded.
1341  */
main(int argc,char ** argv)1342 int main(int argc, char **argv)
1343 {
1344     int rc = 1;
1345     SSL_CTX *ctx = NULL;
1346     int fd = -1;
1347     unsigned long port;
1348 
1349     if (argc < 4) {
1350         fprintf(stderr, "usage: %s <port> <server.crt> <server.key>\n",
1351                 argv[0]);
1352         goto err;
1353     }
1354 
1355     /* Create SSL_CTX. */
1356     if ((ctx = create_ctx(argv[2], argv[3])) == NULL)
1357         goto err;
1358 
1359     /* Parse port number from command line arguments. */
1360     port = strtoul(argv[1], NULL, 0);
1361     if (port == 0 || port > UINT16_MAX) {
1362         fprintf(stderr, "invalid port: %lu\n", port);
1363         goto err;
1364     }
1365 
1366     /* Create UDP socket. */
1367     if ((fd = create_socket((uint16_t)port)) < 0)
1368         goto err;
1369 
1370     /* Enter QUIC server connection acceptance loop. */
1371     if (!run_quic_server(ctx, fd))
1372         goto err;
1373 
1374     rc = 0;
1375 err:
1376     if (rc != 0)
1377         ERR_print_errors_fp(stderr);
1378 
1379     SSL_CTX_free(ctx);
1380 
1381     if (fd != -1)
1382         BIO_closesocket(fd);
1383 
1384     return rc;
1385 }
1386